2024-11-10 18:13:16,401 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 18:13:16,427 main DEBUG Took 0.023142 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 18:13:16,428 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 18:13:16,429 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 18:13:16,430 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 18:13:16,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,448 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 18:13:16,469 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,472 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,473 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,474 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,475 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,475 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,477 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,478 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,478 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,479 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,479 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,480 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,481 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,481 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,482 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,482 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,483 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,483 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,484 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 18:13:16,485 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,485 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 18:13:16,487 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 18:13:16,489 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 18:13:16,491 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 18:13:16,492 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 18:13:16,493 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 18:13:16,494 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 18:13:16,505 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 18:13:16,508 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 18:13:16,511 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 18:13:16,511 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 18:13:16,512 main DEBUG createAppenders(={Console}) 2024-11-10 18:13:16,513 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-10 18:13:16,513 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-10 18:13:16,514 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-10 18:13:16,514 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 18:13:16,515 main DEBUG OutputStream closed 2024-11-10 18:13:16,515 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 18:13:16,515 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 18:13:16,516 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-10 18:13:16,609 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 18:13:16,611 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 18:13:16,613 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 18:13:16,614 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 18:13:16,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 18:13:16,615 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 18:13:16,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 18:13:16,616 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 18:13:16,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 18:13:16,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 18:13:16,617 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 18:13:16,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 18:13:16,621 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 18:13:16,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 18:13:16,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 18:13:16,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 18:13:16,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 18:13:16,623 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 18:13:16,626 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 18:13:16,627 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-10 18:13:16,627 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 18:13:16,628 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-10T18:13:17,001 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61 2024-11-10 18:13:17,005 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 18:13:17,005 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T18:13:17,018 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-10T18:13:17,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-10T18:13:17,081 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4, deleteOnExit=true 2024-11-10T18:13:17,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-10T18:13:17,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/test.cache.data in system properties and HBase conf 2024-11-10T18:13:17,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/hadoop.tmp.dir in system properties and HBase conf 2024-11-10T18:13:17,085 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/hadoop.log.dir in system properties and HBase conf 2024-11-10T18:13:17,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-10T18:13:17,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-10T18:13:17,087 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-10T18:13:17,225 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T18:13:17,369 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T18:13:17,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-10T18:13:17,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-10T18:13:17,376 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-10T18:13:17,377 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T18:13:17,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-10T18:13:17,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-10T18:13:17,379 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-10T18:13:17,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T18:13:17,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-10T18:13:17,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/nfs.dump.dir in system properties and HBase conf 2024-11-10T18:13:17,381 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/java.io.tmpdir in system properties and HBase conf 2024-11-10T18:13:17,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-10T18:13:17,382 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-10T18:13:17,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-10T18:13:18,332 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-10T18:13:18,435 INFO [Time-limited test {}] log.Log(170): Logging initialized @3029ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-10T18:13:18,530 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T18:13:18,604 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T18:13:18,631 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T18:13:18,631 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T18:13:18,633 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-10T18:13:18,652 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T18:13:18,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/hadoop.log.dir/,AVAILABLE} 2024-11-10T18:13:18,661 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T18:13:18,897 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/java.io.tmpdir/jetty-localhost-41265-hadoop-hdfs-3_4_1-tests_jar-_-any-16181059052028738370/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T18:13:18,917 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:41265} 2024-11-10T18:13:18,917 INFO [Time-limited test {}] server.Server(415): Started @3513ms 2024-11-10T18:13:19,367 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-10T18:13:19,374 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-10T18:13:19,376 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-10T18:13:19,377 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-10T18:13:19,377 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-10T18:13:19,381 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/hadoop.log.dir/,AVAILABLE} 2024-11-10T18:13:19,381 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-10T18:13:19,517 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/java.io.tmpdir/jetty-localhost-46147-hadoop-hdfs-3_4_1-tests_jar-_-any-14363089242119503225/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T18:13:19,518 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:46147} 2024-11-10T18:13:19,518 INFO [Time-limited test {}] server.Server(415): Started @4113ms 2024-11-10T18:13:19,583 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-10T18:13:20,279 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/data/data2/current/BP-1691272945-172.17.0.2-1731262398071/current, will proceed with Du for space computation calculation, 2024-11-10T18:13:20,279 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/data/data1/current/BP-1691272945-172.17.0.2-1731262398071/current, will proceed with Du for space computation calculation, 2024-11-10T18:13:20,362 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-10T18:13:20,453 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5386800462e1fbbe with lease ID 0x2c438c7d9c5ed0a2: Processing first storage report for DS-32dc0a6f-8f83-454e-a8fd-e15dd0ac7b7c from datanode DatanodeRegistration(127.0.0.1:39089, datanodeUuid=a583ee22-9a26-4d25-9d41-36495a87b783, infoPort=32985, infoSecurePort=0, ipcPort=33069, storageInfo=lv=-57;cid=testClusterID;nsid=1637440506;c=1731262398071) 2024-11-10T18:13:20,456 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5386800462e1fbbe with lease ID 0x2c438c7d9c5ed0a2: from storage DS-32dc0a6f-8f83-454e-a8fd-e15dd0ac7b7c node DatanodeRegistration(127.0.0.1:39089, datanodeUuid=a583ee22-9a26-4d25-9d41-36495a87b783, infoPort=32985, infoSecurePort=0, ipcPort=33069, storageInfo=lv=-57;cid=testClusterID;nsid=1637440506;c=1731262398071), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-10T18:13:20,457 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5386800462e1fbbe with lease ID 0x2c438c7d9c5ed0a2: Processing first storage report for DS-5ee9a583-62fe-4a14-9333-72a34ee3f17e from datanode DatanodeRegistration(127.0.0.1:39089, datanodeUuid=a583ee22-9a26-4d25-9d41-36495a87b783, infoPort=32985, infoSecurePort=0, ipcPort=33069, storageInfo=lv=-57;cid=testClusterID;nsid=1637440506;c=1731262398071) 2024-11-10T18:13:20,457 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5386800462e1fbbe with lease ID 0x2c438c7d9c5ed0a2: from storage DS-5ee9a583-62fe-4a14-9333-72a34ee3f17e node DatanodeRegistration(127.0.0.1:39089, datanodeUuid=a583ee22-9a26-4d25-9d41-36495a87b783, infoPort=32985, infoSecurePort=0, ipcPort=33069, storageInfo=lv=-57;cid=testClusterID;nsid=1637440506;c=1731262398071), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-10T18:13:20,497 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61 2024-11-10T18:13:20,604 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/zookeeper_0, clientPort=59263, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T18:13:20,616 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59263 2024-11-10T18:13:20,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:20,638 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:20,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741825_1001 (size=7) 2024-11-10T18:13:21,370 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 with version=8 2024-11-10T18:13:21,370 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/hbase-staging 2024-11-10T18:13:21,471 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T18:13:21,753 INFO [Time-limited test {}] client.ConnectionUtils(128): master/691ea56905d2:0 server-side Connection retries=45 2024-11-10T18:13:21,769 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T18:13:21,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T18:13:21,777 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T18:13:21,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T18:13:21,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T18:13:21,946 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T18:13:22,022 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T18:13:22,034 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T18:13:22,039 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T18:13:22,072 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1008 (auto-detected) 2024-11-10T18:13:22,073 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T18:13:22,100 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41343 2024-11-10T18:13:22,131 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41343 connecting to ZooKeeper ensemble=127.0.0.1:59263 2024-11-10T18:13:22,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:413430x0, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T18:13:22,175 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41343-0x1010315b2cd0000 connected 2024-11-10T18:13:22,218 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:22,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:22,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T18:13:22,242 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193, hbase.cluster.distributed=false 2024-11-10T18:13:22,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T18:13:22,273 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41343 2024-11-10T18:13:22,273 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41343 2024-11-10T18:13:22,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41343 2024-11-10T18:13:22,274 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41343 2024-11-10T18:13:22,275 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41343 2024-11-10T18:13:22,409 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/691ea56905d2:0 server-side Connection retries=45 2024-11-10T18:13:22,412 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T18:13:22,412 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T18:13:22,413 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T18:13:22,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T18:13:22,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T18:13:22,417 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T18:13:22,419 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T18:13:22,420 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32909 2024-11-10T18:13:22,422 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32909 connecting to ZooKeeper ensemble=127.0.0.1:59263 2024-11-10T18:13:22,423 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:22,426 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:22,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329090x0, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T18:13:22,434 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:329090x0, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T18:13:22,436 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32909-0x1010315b2cd0001 connected 2024-11-10T18:13:22,439 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T18:13:22,450 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T18:13:22,454 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T18:13:22,460 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T18:13:22,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32909 2024-11-10T18:13:22,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32909 2024-11-10T18:13:22,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32909 2024-11-10T18:13:22,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32909 2024-11-10T18:13:22,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32909 2024-11-10T18:13:22,483 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;691ea56905d2:41343 2024-11-10T18:13:22,484 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/691ea56905d2,41343,1731262401541 2024-11-10T18:13:22,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T18:13:22,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T18:13:22,500 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/691ea56905d2,41343,1731262401541 2024-11-10T18:13:22,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T18:13:22,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:22,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:22,530 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T18:13:22,531 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/691ea56905d2,41343,1731262401541 from backup master directory 2024-11-10T18:13:22,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T18:13:22,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/691ea56905d2,41343,1731262401541 2024-11-10T18:13:22,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T18:13:22,537 WARN [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T18:13:22,538 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=691ea56905d2,41343,1731262401541 2024-11-10T18:13:22,541 INFO [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T18:13:22,543 INFO [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T18:13:22,613 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/hbase.id] with ID: a8ca52b0-39cc-4209-9f20-fdad45deec70 2024-11-10T18:13:22,614 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/.tmp/hbase.id 2024-11-10T18:13:22,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741826_1002 (size=42) 2024-11-10T18:13:23,031 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/.tmp/hbase.id]:[hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/hbase.id] 2024-11-10T18:13:23,077 INFO [master/691ea56905d2:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-10T18:13:23,082 INFO [master/691ea56905d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T18:13:23,103 INFO [master/691ea56905d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-10T18:13:23,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:23,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:23,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741827_1003 (size=196) 2024-11-10T18:13:23,546 INFO [master/691ea56905d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:13:23,549 INFO [master/691ea56905d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T18:13:23,571 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:23,576 INFO [master/691ea56905d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T18:13:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741828_1004 (size=1189) 2024-11-10T18:13:23,629 INFO [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store 2024-11-10T18:13:23,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741829_1005 (size=34) 2024-11-10T18:13:24,056 INFO [master/691ea56905d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T18:13:24,059 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:13:24,061 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T18:13:24,061 INFO [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:13:24,062 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:13:24,064 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T18:13:24,064 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:13:24,064 INFO [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:13:24,065 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731262404061Disabling compacts and flushes for region at 1731262404061Disabling writes for close at 1731262404064 (+3 ms)Writing region close event to WAL at 1731262404064Closed at 1731262404064 2024-11-10T18:13:24,068 WARN [master/691ea56905d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/.initializing 2024-11-10T18:13:24,068 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/WALs/691ea56905d2,41343,1731262401541 2024-11-10T18:13:24,077 INFO [master/691ea56905d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T18:13:24,094 INFO [master/691ea56905d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=691ea56905d2%2C41343%2C1731262401541, suffix=, logDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/WALs/691ea56905d2,41343,1731262401541, archiveDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/oldWALs, maxLogs=10 2024-11-10T18:13:24,130 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/WALs/691ea56905d2,41343,1731262401541/691ea56905d2%2C41343%2C1731262401541.1731262404100, exclude list is [], retry=0 2024-11-10T18:13:24,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39089,DS-32dc0a6f-8f83-454e-a8fd-e15dd0ac7b7c,DISK] 2024-11-10T18:13:24,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-10T18:13:24,202 INFO [master/691ea56905d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/WALs/691ea56905d2,41343,1731262401541/691ea56905d2%2C41343%2C1731262401541.1731262404100 2024-11-10T18:13:24,203 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32985:32985)] 2024-11-10T18:13:24,204 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:13:24,204 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:13:24,208 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,209 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,291 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T18:13:24,295 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:24,298 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:24,299 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T18:13:24,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:24,304 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:13:24,305 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,307 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T18:13:24,308 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:24,309 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:13:24,309 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,312 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T18:13:24,312 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:24,313 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:13:24,314 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,318 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,320 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,326 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,327 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,332 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T18:13:24,338 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T18:13:24,342 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:13:24,344 INFO [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58775065, jitterRate=-0.1241832822561264}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T18:13:24,350 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731262404223Initializing all the Stores at 1731262404225 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262404226 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262404227 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262404227Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262404227Cleaning up temporary data from old regions at 1731262404328 (+101 ms)Region opened successfully at 1731262404349 (+21 ms) 2024-11-10T18:13:24,351 INFO [master/691ea56905d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T18:13:24,388 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d67c8de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=691ea56905d2/172.17.0.2:0 2024-11-10T18:13:24,422 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T18:13:24,434 INFO [master/691ea56905d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T18:13:24,434 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T18:13:24,437 INFO [master/691ea56905d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T18:13:24,439 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-10T18:13:24,444 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-10T18:13:24,444 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T18:13:24,472 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T18:13:24,484 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T18:13:24,486 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T18:13:24,489 INFO [master/691ea56905d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T18:13:24,490 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T18:13:24,492 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T18:13:24,494 INFO [master/691ea56905d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T18:13:24,498 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T18:13:24,500 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T18:13:24,501 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T18:13:24,503 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T18:13:24,522 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T18:13:24,524 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T18:13:24,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T18:13:24,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T18:13:24,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:24,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:24,531 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=691ea56905d2,41343,1731262401541, sessionid=0x1010315b2cd0000, setting cluster-up flag (Was=false) 2024-11-10T18:13:24,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:24,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:24,553 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T18:13:24,555 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=691ea56905d2,41343,1731262401541 2024-11-10T18:13:24,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:24,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:24,567 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T18:13:24,568 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=691ea56905d2,41343,1731262401541 2024-11-10T18:13:24,575 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T18:13:24,669 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T18:13:24,683 INFO [master/691ea56905d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T18:13:24,690 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(746): ClusterId : a8ca52b0-39cc-4209-9f20-fdad45deec70 2024-11-10T18:13:24,694 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T18:13:24,695 INFO [master/691ea56905d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T18:13:24,699 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T18:13:24,700 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T18:13:24,703 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T18:13:24,704 DEBUG [RS:0;691ea56905d2:32909 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492d8165, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=691ea56905d2/172.17.0.2:0 2024-11-10T18:13:24,703 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 691ea56905d2,41343,1731262401541 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T18:13:24,713 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/691ea56905d2:0, corePoolSize=5, maxPoolSize=5 2024-11-10T18:13:24,713 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/691ea56905d2:0, corePoolSize=5, maxPoolSize=5 2024-11-10T18:13:24,714 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/691ea56905d2:0, corePoolSize=5, maxPoolSize=5 2024-11-10T18:13:24,714 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/691ea56905d2:0, corePoolSize=5, maxPoolSize=5 2024-11-10T18:13:24,714 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/691ea56905d2:0, corePoolSize=10, maxPoolSize=10 2024-11-10T18:13:24,714 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,715 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/691ea56905d2:0, corePoolSize=2, maxPoolSize=2 2024-11-10T18:13:24,715 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,723 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731262434723 2024-11-10T18:13:24,725 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T18:13:24,726 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T18:13:24,728 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T18:13:24,728 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T18:13:24,731 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T18:13:24,731 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T18:13:24,732 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T18:13:24,732 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T18:13:24,733 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;691ea56905d2:32909 2024-11-10T18:13:24,732 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,737 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T18:13:24,737 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:24,738 INFO [RS:0;691ea56905d2:32909 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T18:13:24,738 INFO [RS:0;691ea56905d2:32909 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T18:13:24,737 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T18:13:24,738 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T18:13:24,738 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T18:13:24,738 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T18:13:24,740 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T18:13:24,741 INFO [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T18:13:24,741 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(2659): reportForDuty to master=691ea56905d2,41343,1731262401541 with port=32909, startcode=1731262402364 2024-11-10T18:13:24,743 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/691ea56905d2:0:becomeActiveMaster-HFileCleaner.large.0-1731262404742,5,FailOnTimeoutGroup] 2024-11-10T18:13:24,743 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/691ea56905d2:0:becomeActiveMaster-HFileCleaner.small.0-1731262404743,5,FailOnTimeoutGroup] 2024-11-10T18:13:24,744 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,744 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T18:13:24,745 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,745 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741831_1007 (size=1321) 2024-11-10T18:13:24,754 DEBUG [RS:0;691ea56905d2:32909 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T18:13:24,793 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45191, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T18:13:24,799 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41343 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 691ea56905d2,32909,1731262402364 2024-11-10T18:13:24,802 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41343 {}] master.ServerManager(517): Registering regionserver=691ea56905d2,32909,1731262402364 2024-11-10T18:13:24,817 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:13:24,817 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39555 2024-11-10T18:13:24,817 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T18:13:24,823 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T18:13:24,824 DEBUG [RS:0;691ea56905d2:32909 {}] zookeeper.ZKUtil(111): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/691ea56905d2,32909,1731262402364 2024-11-10T18:13:24,824 WARN [RS:0;691ea56905d2:32909 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T18:13:24,824 INFO [RS:0;691ea56905d2:32909 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T18:13:24,824 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364 2024-11-10T18:13:24,826 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [691ea56905d2,32909,1731262402364] 2024-11-10T18:13:24,851 INFO [RS:0;691ea56905d2:32909 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T18:13:24,865 INFO [RS:0;691ea56905d2:32909 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T18:13:24,870 INFO [RS:0;691ea56905d2:32909 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T18:13:24,870 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,871 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T18:13:24,877 INFO [RS:0;691ea56905d2:32909 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T18:13:24,878 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,879 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,879 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,879 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,879 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,879 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,879 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/691ea56905d2:0, corePoolSize=2, maxPoolSize=2 2024-11-10T18:13:24,880 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,880 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,880 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,880 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,881 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,881 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/691ea56905d2:0, corePoolSize=1, maxPoolSize=1 2024-11-10T18:13:24,881 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/691ea56905d2:0, corePoolSize=3, maxPoolSize=3 2024-11-10T18:13:24,881 DEBUG [RS:0;691ea56905d2:32909 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0, corePoolSize=3, maxPoolSize=3 2024-11-10T18:13:24,883 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,883 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,883 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,883 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,883 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,884 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,32909,1731262402364-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T18:13:24,903 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T18:13:24,905 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,32909,1731262402364-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,906 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,906 INFO [RS:0;691ea56905d2:32909 {}] regionserver.Replication(171): 691ea56905d2,32909,1731262402364 started 2024-11-10T18:13:24,926 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:24,926 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1482): Serving as 691ea56905d2,32909,1731262402364, RpcServer on 691ea56905d2/172.17.0.2:32909, sessionid=0x1010315b2cd0001 2024-11-10T18:13:24,927 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T18:13:24,927 DEBUG [RS:0;691ea56905d2:32909 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 691ea56905d2,32909,1731262402364 2024-11-10T18:13:24,927 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '691ea56905d2,32909,1731262402364' 2024-11-10T18:13:24,927 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T18:13:24,929 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T18:13:24,929 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T18:13:24,929 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T18:13:24,929 DEBUG [RS:0;691ea56905d2:32909 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 691ea56905d2,32909,1731262402364 2024-11-10T18:13:24,930 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '691ea56905d2,32909,1731262402364' 2024-11-10T18:13:24,930 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T18:13:24,930 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T18:13:24,931 DEBUG [RS:0;691ea56905d2:32909 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T18:13:24,931 INFO [RS:0;691ea56905d2:32909 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T18:13:24,931 INFO [RS:0;691ea56905d2:32909 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T18:13:25,037 INFO [RS:0;691ea56905d2:32909 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T18:13:25,040 INFO [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=691ea56905d2%2C32909%2C1731262402364, suffix=, logDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364, archiveDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/oldWALs, maxLogs=32 2024-11-10T18:13:25,057 DEBUG [RS:0;691ea56905d2:32909 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364/691ea56905d2%2C32909%2C1731262402364.1731262405043, exclude list is [], retry=0 2024-11-10T18:13:25,062 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39089,DS-32dc0a6f-8f83-454e-a8fd-e15dd0ac7b7c,DISK] 2024-11-10T18:13:25,065 INFO [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364/691ea56905d2%2C32909%2C1731262402364.1731262405043 2024-11-10T18:13:25,066 DEBUG [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32985:32985)] 2024-11-10T18:13:25,154 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T18:13:25,154 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:13:25,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741833_1009 (size=32) 2024-11-10T18:13:25,566 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:13:25,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T18:13:25,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T18:13:25,571 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:25,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:25,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T18:13:25,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T18:13:25,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:25,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:25,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T18:13:25,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T18:13:25,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:25,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:25,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T18:13:25,581 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T18:13:25,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:25,582 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:25,583 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T18:13:25,584 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740 2024-11-10T18:13:25,585 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740 2024-11-10T18:13:25,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T18:13:25,587 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T18:13:25,588 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:13:25,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T18:13:25,594 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:13:25,595 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59556359, jitterRate=-0.1125410944223404}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:13:25,597 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731262405566Initializing all the Stores at 1731262405568 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262405568Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262405568Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262405568Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262405568Cleaning up temporary data from old regions at 1731262405587 (+19 ms)Region opened successfully at 1731262405597 (+10 ms) 2024-11-10T18:13:25,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T18:13:25,598 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T18:13:25,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T18:13:25,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T18:13:25,598 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T18:13:25,599 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T18:13:25,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731262405598Disabling compacts and flushes for region at 1731262405598Disabling writes for close at 1731262405598Writing region close event to WAL at 1731262405599 (+1 ms)Closed at 1731262405599 2024-11-10T18:13:25,602 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T18:13:25,603 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T18:13:25,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T18:13:25,618 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T18:13:25,621 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T18:13:25,774 DEBUG [691ea56905d2:41343 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-10T18:13:25,787 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:13:25,794 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 691ea56905d2,32909,1731262402364, state=OPENING 2024-11-10T18:13:25,799 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T18:13:25,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:25,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:13:25,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T18:13:25,802 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T18:13:25,804 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T18:13:25,806 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:13:25,982 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T18:13:25,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59253, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T18:13:25,995 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T18:13:25,995 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T18:13:25,996 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T18:13:25,999 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=691ea56905d2%2C32909%2C1731262402364.meta, suffix=.meta, logDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364, archiveDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/oldWALs, maxLogs=32 2024-11-10T18:13:26,017 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364/691ea56905d2%2C32909%2C1731262402364.meta.1731262406001.meta, exclude list is [], retry=0 2024-11-10T18:13:26,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39089,DS-32dc0a6f-8f83-454e-a8fd-e15dd0ac7b7c,DISK] 2024-11-10T18:13:26,024 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/WALs/691ea56905d2,32909,1731262402364/691ea56905d2%2C32909%2C1731262402364.meta.1731262406001.meta 2024-11-10T18:13:26,024 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:32985:32985)] 2024-11-10T18:13:26,025 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:13:26,026 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T18:13:26,029 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T18:13:26,034 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T18:13:26,039 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T18:13:26,040 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:13:26,040 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T18:13:26,040 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T18:13:26,043 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-10T18:13:26,045 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T18:13:26,045 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:26,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:26,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-10T18:13:26,048 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T18:13:26,048 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:26,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:26,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-10T18:13:26,050 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T18:13:26,051 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:26,051 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:26,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-10T18:13:26,053 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T18:13:26,053 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:26,054 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T18:13:26,054 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T18:13:26,055 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740 2024-11-10T18:13:26,058 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740 2024-11-10T18:13:26,060 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T18:13:26,060 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T18:13:26,060 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:13:26,063 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T18:13:26,064 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74931110, jitterRate=0.11656054854393005}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:13:26,064 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T18:13:26,066 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731262406041Writing region info on filesystem at 1731262406041Initializing all the Stores at 1731262406043 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262406043Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262406043Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262406043Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731262406043Cleaning up temporary data from old regions at 1731262406060 (+17 ms)Running coprocessor post-open hooks at 1731262406065 (+5 ms)Region opened successfully at 1731262406066 (+1 ms) 2024-11-10T18:13:26,073 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731262405972 2024-11-10T18:13:26,084 DEBUG [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T18:13:26,085 INFO [RS_OPEN_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T18:13:26,087 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:13:26,088 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 691ea56905d2,32909,1731262402364, state=OPEN 2024-11-10T18:13:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T18:13:26,095 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T18:13:26,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T18:13:26,095 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T18:13:26,095 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:26,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T18:13:26,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=691ea56905d2,32909,1731262402364 in 290 msec 2024-11-10T18:13:26,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T18:13:26,107 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 493 msec 2024-11-10T18:13:26,108 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T18:13:26,108 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T18:13:26,137 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:26,139 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:26,163 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:26,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57899, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:26,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5840 sec 2024-11-10T18:13:26,188 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731262406188, completionTime=-1 2024-11-10T18:13:26,191 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-10T18:13:26,191 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-10T18:13:26,221 INFO [master/691ea56905d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-10T18:13:26,221 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731262466221 2024-11-10T18:13:26,221 INFO [master/691ea56905d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731262526221 2024-11-10T18:13:26,221 INFO [master/691ea56905d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-10T18:13:26,224 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,41343,1731262401541-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,224 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,41343,1731262401541-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,224 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,41343,1731262401541-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,226 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-691ea56905d2:41343, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,226 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,227 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,234 DEBUG [master/691ea56905d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T18:13:26,259 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.721sec 2024-11-10T18:13:26,261 INFO [master/691ea56905d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T18:13:26,262 INFO [master/691ea56905d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T18:13:26,263 INFO [master/691ea56905d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T18:13:26,263 INFO [master/691ea56905d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T18:13:26,264 INFO [master/691ea56905d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T18:13:26,264 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,41343,1731262401541-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T18:13:26,265 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,41343,1731262401541-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T18:13:26,273 DEBUG [master/691ea56905d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T18:13:26,274 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T18:13:26,274 INFO [master/691ea56905d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=691ea56905d2,41343,1731262401541-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T18:13:26,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:26,302 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T18:13:26,302 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T18:13:26,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:26,308 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:26,316 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:26,319 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f93b2f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:26,327 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:26,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:26,331 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:26,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:26,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:26,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:26,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=691ea56905d2,41343,1731262401541 2024-11-10T18:13:26,391 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=267, ProcessCount=11, AvailableMemoryMB=8747 2024-11-10T18:13:26,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:13:26,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:13:26,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a16ee17 2024-11-10T18:13:26,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:13:26,418 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:13:26,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:13:26,425 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:13:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:26,437 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T18:13:26,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-10T18:13:26,440 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:26,442 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T18:13:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T18:13:26,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741835_1011 (size=1061) 2024-11-10T18:13:26,458 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:13:26,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741836_1012 (size=53) 2024-11-10T18:13:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T18:13:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T18:13:26,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:13:26,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 17351107c8cbd49315660faf09692c94, disabling compactions & flushes 2024-11-10T18:13:26,868 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:26,868 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. after waiting 0 ms 2024-11-10T18:13:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:26,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:26,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 17351107c8cbd49315660faf09692c94: Waiting for close lock at 1731262406868Disabling compacts and flushes for region at 1731262406868Disabling writes for close at 1731262406869 (+1 ms)Writing region close event to WAL at 1731262406869Closed at 1731262406869 2024-11-10T18:13:26,871 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T18:13:26,878 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731262406871"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731262406871"}]},"ts":"1731262406871"} 2024-11-10T18:13:26,883 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T18:13:26,885 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T18:13:26,888 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262406885"}]},"ts":"1731262406885"} 2024-11-10T18:13:26,892 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T18:13:26,894 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, ASSIGN}] 2024-11-10T18:13:26,896 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, ASSIGN 2024-11-10T18:13:26,897 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, ASSIGN; state=OFFLINE, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=false 2024-11-10T18:13:27,049 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=17351107c8cbd49315660faf09692c94, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:13:27,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, ASSIGN because future has completed 2024-11-10T18:13:27,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:13:27,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T18:13:27,215 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:27,216 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:13:27,216 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,217 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:13:27,217 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,217 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,220 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,223 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:13:27,224 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17351107c8cbd49315660faf09692c94 columnFamilyName A 2024-11-10T18:13:27,224 DEBUG [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:27,225 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.HStore(327): Store=17351107c8cbd49315660faf09692c94/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:13:27,225 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,227 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:13:27,227 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17351107c8cbd49315660faf09692c94 columnFamilyName B 2024-11-10T18:13:27,227 DEBUG [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:27,228 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.HStore(327): Store=17351107c8cbd49315660faf09692c94/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:13:27,228 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,230 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:13:27,230 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17351107c8cbd49315660faf09692c94 columnFamilyName C 2024-11-10T18:13:27,230 DEBUG [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:13:27,231 INFO [StoreOpener-17351107c8cbd49315660faf09692c94-1 {}] regionserver.HStore(327): Store=17351107c8cbd49315660faf09692c94/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:13:27,232 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:27,232 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,234 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,234 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,237 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,237 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,238 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:13:27,240 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,244 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:13:27,245 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 17351107c8cbd49315660faf09692c94; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72780027, jitterRate=0.08450691401958466}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:13:27,246 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,247 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 17351107c8cbd49315660faf09692c94: Running coprocessor pre-open hook at 1731262407217Writing region info on filesystem at 1731262407217Initializing all the Stores at 1731262407219 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262407219Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262407219Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262407219Cleaning up temporary data from old regions at 1731262407237 (+18 ms)Running coprocessor post-open hooks at 1731262407246 (+9 ms)Region opened successfully at 1731262407246 2024-11-10T18:13:27,249 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., pid=6, masterSystemTime=1731262407209 2024-11-10T18:13:27,253 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:27,253 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:27,254 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=17351107c8cbd49315660faf09692c94, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:13:27,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:13:27,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T18:13:27,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 in 204 msec 2024-11-10T18:13:27,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T18:13:27,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, ASSIGN in 369 msec 2024-11-10T18:13:27,269 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T18:13:27,270 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262407269"}]},"ts":"1731262407269"} 2024-11-10T18:13:27,273 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T18:13:27,274 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T18:13:27,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 846 msec 2024-11-10T18:13:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T18:13:27,586 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:27,592 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,592 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,592 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,594 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,594 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29446fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,597 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e077d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,600 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,604 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:13:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:13:27,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d4e458b 2024-11-10T18:13:27,608 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:13:27,610 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:13:27,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,615 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,616 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,616 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,619 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,622 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,625 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33630, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,627 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,627 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,627 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,629 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63181207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,632 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33648, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,634 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,635 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dac934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,642 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e3ab4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,644 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c4ddcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38cde2e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c7bd7ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,653 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44731cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,655 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@870009d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,657 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,658 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d95629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,660 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3837915e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,661 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:13:27,661 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:13:27,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:13:27,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:13:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:13:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e873e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:13:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:13:27,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:27,664 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:13:27,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21616163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:13:27,670 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,670 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,670 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,670 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,671 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,671 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,671 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,671 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,672 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:13:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,675 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,677 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:13:27,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40444, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:13:27,681 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40446, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40490, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40472, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40450, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,683 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:13:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-10T18:13:27,686 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T18:13:27,689 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:27,690 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,690 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:13:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:27,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:13:27,775 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:27,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:27,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:27,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:27,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:27,777 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T18:13:27,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:27,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:27,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:27,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:27,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:27,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/5e094b6875c141d48747ffa8d2ffa922 is 50, key is test_row_0/A:col10/1731262407754/Put/seqid=0 2024-11-10T18:13:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:27,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741837_1013 (size=9657) 2024-11-10T18:13:27,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/5e094b6875c141d48747ffa8d2ffa922 2024-11-10T18:13:27,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262467938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:27,968 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262407938, max timestamp=1731262407950Num uniques 859; 2024-11-10T18:13:27,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:27,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262407938, max timestamp=1731262407950Num uniques 859; 2024-11-10T18:13:27,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:27,979 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262407938, max timestamp=1731262407950Num uniques 859; 2024-11-10T18:13:27,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:27,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:27,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T18:13:28,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/07fd4a36d5e44a39ab7aae28872ab027 is 50, key is test_row_0/B:col10/1731262407754/Put/seqid=0 2024-11-10T18:13:28,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741838_1014 (size=9657) 2024-11-10T18:13:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:28,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/07fd4a36d5e44a39ab7aae28872ab027 2024-11-10T18:13:28,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262468034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262468038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262468051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262468054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/5d39316a66ad4403928a0b913dcc61d1 is 50, key is test_row_0/C:col10/1731262407754/Put/seqid=0 2024-11-10T18:13:28,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262468106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741839_1015 (size=9657) 2024-11-10T18:13:28,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/5d39316a66ad4403928a0b913dcc61d1 2024-11-10T18:13:28,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/5e094b6875c141d48747ffa8d2ffa922 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/5e094b6875c141d48747ffa8d2ffa922 2024-11-10T18:13:28,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/5e094b6875c141d48747ffa8d2ffa922, entries=100, sequenceid=13, filesize=9.4 K 2024-11-10T18:13:28,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/07fd4a36d5e44a39ab7aae28872ab027 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/07fd4a36d5e44a39ab7aae28872ab027 2024-11-10T18:13:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262468156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/07fd4a36d5e44a39ab7aae28872ab027, entries=100, sequenceid=13, filesize=9.4 K 2024-11-10T18:13:28,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/5d39316a66ad4403928a0b913dcc61d1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/5d39316a66ad4403928a0b913dcc61d1 2024-11-10T18:13:28,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262468165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262468175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,204 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/5d39316a66ad4403928a0b913dcc61d1, entries=100, sequenceid=13, filesize=9.4 K 2024-11-10T18:13:28,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262468176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:28,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 17351107c8cbd49315660faf09692c94 in 444ms, sequenceid=13, compaction requested=false 2024-11-10T18:13:28,215 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T18:13:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:28,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T18:13:28,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:28,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T18:13:28,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:28,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/a9e06b171a0041008ea276dd1887c1c6 is 50, key is test_row_0/A:col10/1731262408045/Put/seqid=0 2024-11-10T18:13:28,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:28,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741840_1016 (size=14341) 2024-11-10T18:13:28,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/a9e06b171a0041008ea276dd1887c1c6 2024-11-10T18:13:28,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262468412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262468417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262468417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262468418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262468417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/860deb6bd2774f6bbe3e70e8db373855 is 50, key is test_row_0/B:col10/1731262408045/Put/seqid=0 2024-11-10T18:13:28,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741841_1017 (size=12001) 2024-11-10T18:13:28,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/860deb6bd2774f6bbe3e70e8db373855 2024-11-10T18:13:28,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/61209753cc034d3da68b30a8512bd6bd is 50, key is test_row_0/C:col10/1731262408045/Put/seqid=0 2024-11-10T18:13:28,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741842_1018 (size=12001) 2024-11-10T18:13:28,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:28,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262468535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262468536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262468735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262468736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262468736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262468755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262468756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T18:13:28,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:28,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:28,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/61209753cc034d3da68b30a8512bd6bd 2024-11-10T18:13:28,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/a9e06b171a0041008ea276dd1887c1c6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/a9e06b171a0041008ea276dd1887c1c6 2024-11-10T18:13:28,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/a9e06b171a0041008ea276dd1887c1c6, entries=200, sequenceid=50, filesize=14.0 K 2024-11-10T18:13:28,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/860deb6bd2774f6bbe3e70e8db373855 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/860deb6bd2774f6bbe3e70e8db373855 2024-11-10T18:13:28,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/860deb6bd2774f6bbe3e70e8db373855, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T18:13:28,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/61209753cc034d3da68b30a8512bd6bd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/61209753cc034d3da68b30a8512bd6bd 2024-11-10T18:13:28,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/61209753cc034d3da68b30a8512bd6bd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-10T18:13:28,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 17351107c8cbd49315660faf09692c94 in 650ms, sequenceid=50, compaction requested=false 2024-11-10T18:13:28,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:28,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-10T18:13:28,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:28,997 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:13:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:28,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:28,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/16975c232c0b474b8684272d25352890 is 50, key is test_row_0/A:col10/1731262408404/Put/seqid=0 2024-11-10T18:13:29,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741843_1019 (size=12001) 2024-11-10T18:13:29,039 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/16975c232c0b474b8684272d25352890 2024-11-10T18:13:29,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/3084c864878848b99d5820f4f5a93b5b is 50, key is test_row_0/B:col10/1731262408404/Put/seqid=0 2024-11-10T18:13:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:29,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741844_1020 (size=12001) 2024-11-10T18:13:29,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262409137, max timestamp=1731262409159Num uniques 859; 2024-11-10T18:13:29,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:29,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262409137, max timestamp=1731262409159Num uniques 859; 2024-11-10T18:13:29,179 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:29,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262409137, max timestamp=1731262409171Num uniques 859; 2024-11-10T18:13:29,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:29,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262469256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262469258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,277 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=185, cellSize=8470, totalHeapSize=19480, min timestamp=1731262409250, max timestamp=1731262409266Num uniques 1018; 2024-11-10T18:13:29,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731262409250, max timestamp=1731262409266Num uniques 1018; 2024-11-10T18:13:29,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=8470, suffix heap size=133840,new segment heap size=19480 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=185 2024-11-10T18:13:29,279 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T18:13:29,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731262409250, max timestamp=1731262409266Num uniques 1018; 2024-11-10T18:13:29,284 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T18:13:29,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731262409340, max timestamp=1731262409356Num uniques 1027; 2024-11-10T18:13:29,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731262409340, max timestamp=1731262409356Num uniques 1027; 2024-11-10T18:13:29,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T18:13:29,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T18:13:29,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731262409340, max timestamp=1731262409356Num uniques 1044; 2024-11-10T18:13:29,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47814, new segment data size=8102, suffix heap size=136544,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1044, new segment cells count=177 2024-11-10T18:13:29,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262469421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,435 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731262409409, max timestamp=1731262409427Num uniques 1036; 2024-11-10T18:13:29,436 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-10T18:13:29,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262469426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731262409409, max timestamp=1731262409427Num uniques 1036; 2024-11-10T18:13:29,441 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-10T18:13:29,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731262409409, max timestamp=1731262409427Num uniques 1036; 2024-11-10T18:13:29,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-10T18:13:29,499 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/3084c864878848b99d5820f4f5a93b5b 2024-11-10T18:13:29,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262469512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/8992cfd18283428f9e17310b4977fe4c is 50, key is test_row_0/C:col10/1731262408404/Put/seqid=0 2024-11-10T18:13:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,520 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262469515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741845_1021 (size=12001) 2024-11-10T18:13:29,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262469545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,550 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/8992cfd18283428f9e17310b4977fe4c 2024-11-10T18:13:29,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262469545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/16975c232c0b474b8684272d25352890 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/16975c232c0b474b8684272d25352890 2024-11-10T18:13:29,585 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/16975c232c0b474b8684272d25352890, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T18:13:29,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/3084c864878848b99d5820f4f5a93b5b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/3084c864878848b99d5820f4f5a93b5b 2024-11-10T18:13:29,603 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/3084c864878848b99d5820f4f5a93b5b, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T18:13:29,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/8992cfd18283428f9e17310b4977fe4c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8992cfd18283428f9e17310b4977fe4c 2024-11-10T18:13:29,621 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8992cfd18283428f9e17310b4977fe4c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T18:13:29,623 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 17351107c8cbd49315660faf09692c94 in 626ms, sequenceid=60, compaction requested=true 2024-11-10T18:13:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:29,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-10T18:13:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-10T18:13:29,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T18:13:29,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9370 sec 2024-11-10T18:13:29,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=147.60 KB heapSize=408.23 KB 2024-11-10T18:13:29,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:29,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:29,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:29,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:29,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:29,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:29,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:29,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.9590 sec 2024-11-10T18:13:29,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3c04f9f74e7d493a9c2cfb32156d8461 is 50, key is test_row_0/A:col10/1731262409508/Put/seqid=0 2024-11-10T18:13:29,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741846_1022 (size=12151) 2024-11-10T18:13:29,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262469663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,673 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,673 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3c04f9f74e7d493a9c2cfb32156d8461 2024-11-10T18:13:29,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262469667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/becd386b51a047ee87b3a780cc1fdf6f is 50, key is test_row_0/B:col10/1731262409508/Put/seqid=0 2024-11-10T18:13:29,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741847_1023 (size=12151) 2024-11-10T18:13:29,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262469755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262469755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262469775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262469786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-10T18:13:29,835 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:29,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-10T18:13:29,841 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T18:13:29,845 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:29,845 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:29,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T18:13:29,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:29,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262469985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:29,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262469995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T18:13:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262470065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,076 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262470066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,117 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/becd386b51a047ee87b3a780cc1fdf6f 2024-11-10T18:13:30,140 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/f80dfecb2d054ca0abf128024efd62c1 is 50, key is test_row_0/C:col10/1731262409508/Put/seqid=0 2024-11-10T18:13:30,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741848_1024 (size=12151) 2024-11-10T18:13:30,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T18:13:30,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T18:13:30,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262470286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,293 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:30,294 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262470305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262470305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T18:13:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:30,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T18:13:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:30,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T18:13:30,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/f80dfecb2d054ca0abf128024efd62c1 2024-11-10T18:13:30,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3c04f9f74e7d493a9c2cfb32156d8461 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3c04f9f74e7d493a9c2cfb32156d8461 2024-11-10T18:13:30,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3c04f9f74e7d493a9c2cfb32156d8461, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T18:13:30,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/becd386b51a047ee87b3a780cc1fdf6f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/becd386b51a047ee87b3a780cc1fdf6f 2024-11-10T18:13:30,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262470585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262470585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:30,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:30,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/becd386b51a047ee87b3a780cc1fdf6f, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T18:13:30,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/f80dfecb2d054ca0abf128024efd62c1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/f80dfecb2d054ca0abf128024efd62c1 2024-11-10T18:13:30,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/f80dfecb2d054ca0abf128024efd62c1, entries=150, sequenceid=147, filesize=11.9 K 2024-11-10T18:13:30,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~408.19 KB/417984, currentSize=40.25 KB/41220 for 17351107c8cbd49315660faf09692c94 in 975ms, sequenceid=147, compaction requested=true 2024-11-10T18:13:30,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:30,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:30,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:30,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:30,624 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:30,625 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:30,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-10T18:13:30,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,629 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T18:13:30,629 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:30,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:30,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:30,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:30,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:30,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:30,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:30,631 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:30,632 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,634 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/07fd4a36d5e44a39ab7aae28872ab027, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/860deb6bd2774f6bbe3e70e8db373855, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/3084c864878848b99d5820f4f5a93b5b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/becd386b51a047ee87b3a780cc1fdf6f] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=44.7 K 2024-11-10T18:13:30,635 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48150 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:30,636 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:30,636 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,636 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 07fd4a36d5e44a39ab7aae28872ab027, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731262407754 2024-11-10T18:13:30,636 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/5e094b6875c141d48747ffa8d2ffa922, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/a9e06b171a0041008ea276dd1887c1c6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/16975c232c0b474b8684272d25352890, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3c04f9f74e7d493a9c2cfb32156d8461] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=47.0 K 2024-11-10T18:13:30,637 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 860deb6bd2774f6bbe3e70e8db373855, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731262408009 2024-11-10T18:13:30,638 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3084c864878848b99d5820f4f5a93b5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262408390 2024-11-10T18:13:30,639 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting becd386b51a047ee87b3a780cc1fdf6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731262409508 2024-11-10T18:13:30,642 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e094b6875c141d48747ffa8d2ffa922, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731262407754 2024-11-10T18:13:30,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/7983f93b692f421ca1aa14196cf3b2ac is 50, key is test_row_0/A:col10/1731262409655/Put/seqid=0 2024-11-10T18:13:30,645 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e06b171a0041008ea276dd1887c1c6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731262408009 2024-11-10T18:13:30,646 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16975c232c0b474b8684272d25352890, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262408390 2024-11-10T18:13:30,647 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c04f9f74e7d493a9c2cfb32156d8461, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731262409508 2024-11-10T18:13:30,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741849_1025 (size=12151) 2024-11-10T18:13:30,701 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#14 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:30,701 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#13 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:30,703 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/f85dce42f3784f1c868d09695339b4f7 is 50, key is test_row_0/A:col10/1731262409508/Put/seqid=0 2024-11-10T18:13:30,704 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/ebe23100885e471bb8a6b1580b158358 is 50, key is test_row_0/B:col10/1731262409508/Put/seqid=0 2024-11-10T18:13:30,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741851_1027 (size=12289) 2024-11-10T18:13:30,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741850_1026 (size=12289) 2024-11-10T18:13:30,740 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/f85dce42f3784f1c868d09695339b4f7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f85dce42f3784f1c868d09695339b4f7 2024-11-10T18:13:30,766 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into f85dce42f3784f1c868d09695339b4f7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:30,767 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:30,769 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=12, startTime=1731262410612; duration=0sec 2024-11-10T18:13:30,770 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:30,770 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:30,770 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:30,774 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45810 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:30,774 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:30,774 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:30,774 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/5d39316a66ad4403928a0b913dcc61d1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/61209753cc034d3da68b30a8512bd6bd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8992cfd18283428f9e17310b4977fe4c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/f80dfecb2d054ca0abf128024efd62c1] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=44.7 K 2024-11-10T18:13:30,775 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d39316a66ad4403928a0b913dcc61d1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731262407754 2024-11-10T18:13:30,776 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61209753cc034d3da68b30a8512bd6bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731262408009 2024-11-10T18:13:30,777 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8992cfd18283428f9e17310b4977fe4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262408390 2024-11-10T18:13:30,777 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80dfecb2d054ca0abf128024efd62c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731262409508 2024-11-10T18:13:30,803 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#15 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:30,805 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/d196d4a096694c52b567cb5d67350821 is 50, key is test_row_0/C:col10/1731262409508/Put/seqid=0 2024-11-10T18:13:30,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:30,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:30,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741852_1028 (size=12289) 2024-11-10T18:13:30,844 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T18:13:30,849 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/d196d4a096694c52b567cb5d67350821 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d196d4a096694c52b567cb5d67350821 2024-11-10T18:13:30,868 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into d196d4a096694c52b567cb5d67350821(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:30,868 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:30,868 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=12, startTime=1731262410621; duration=0sec 2024-11-10T18:13:30,868 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:30,869 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:30,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262410890, max timestamp=1731262410924Num uniques 859; 2024-11-10T18:13:30,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:30,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262410890, max timestamp=1731262410924Num uniques 859; 2024-11-10T18:13:30,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:30,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262410890, max timestamp=1731262410924Num uniques 859; 2024-11-10T18:13:30,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:30,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T18:13:31,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731262410989, max timestamp=1731262411011Num uniques 1018; 2024-11-10T18:13:31,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731262410989, max timestamp=1731262411011Num uniques 1018; 2024-11-10T18:13:31,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T18:13:31,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T18:13:31,016 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1731262410989, max timestamp=1731262411011Num uniques 1018; 2024-11-10T18:13:31,017 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-11-10T18:13:31,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262471076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,087 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/7983f93b692f421ca1aa14196cf3b2ac 2024-11-10T18:13:31,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262471082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,092 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b31ef52b87e643cf8074871bdf708ceb is 50, key is test_row_0/B:col10/1731262409655/Put/seqid=0 2024-11-10T18:13:31,138 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/ebe23100885e471bb8a6b1580b158358 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ebe23100885e471bb8a6b1580b158358 2024-11-10T18:13:31,152 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into ebe23100885e471bb8a6b1580b158358(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:31,152 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:31,152 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=12, startTime=1731262410620; duration=0sec 2024-11-10T18:13:31,152 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:31,152 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:31,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741853_1029 (size=12151) 2024-11-10T18:13:31,162 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b31ef52b87e643cf8074871bdf708ceb 2024-11-10T18:13:31,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/82161791e53242aca78fc678ca7bcccb is 50, key is test_row_0/C:col10/1731262409655/Put/seqid=0 2024-11-10T18:13:31,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262471195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262471196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741854_1030 (size=12151) 2024-11-10T18:13:31,206 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/82161791e53242aca78fc678ca7bcccb 2024-11-10T18:13:31,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/7983f93b692f421ca1aa14196cf3b2ac as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/7983f93b692f421ca1aa14196cf3b2ac 2024-11-10T18:13:31,240 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/7983f93b692f421ca1aa14196cf3b2ac, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T18:13:31,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b31ef52b87e643cf8074871bdf708ceb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b31ef52b87e643cf8074871bdf708ceb 2024-11-10T18:13:31,253 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b31ef52b87e643cf8074871bdf708ceb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T18:13:31,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/82161791e53242aca78fc678ca7bcccb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82161791e53242aca78fc678ca7bcccb 2024-11-10T18:13:31,288 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82161791e53242aca78fc678ca7bcccb, entries=150, sequenceid=156, filesize=11.9 K 2024-11-10T18:13:31,290 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 17351107c8cbd49315660faf09692c94 in 660ms, sequenceid=156, compaction requested=false 2024-11-10T18:13:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:31,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-10T18:13:31,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-10T18:13:31,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-10T18:13:31,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4490 sec 2024-11-10T18:13:31,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.4680 sec 2024-11-10T18:13:31,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:31,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-11-10T18:13:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:31,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:31,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:31,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/15b03b5a2a8b48acbf80aca844e45361 is 50, key is test_row_0/A:col10/1731262411408/Put/seqid=0 2024-11-10T18:13:31,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741855_1031 (size=12151) 2024-11-10T18:13:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/15b03b5a2a8b48acbf80aca844e45361 2024-11-10T18:13:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262471434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262471439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,446 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/a486eefd61e14512be3666cfe5800bd8 is 50, key is test_row_0/B:col10/1731262411408/Put/seqid=0 2024-11-10T18:13:31,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741856_1032 (size=12151) 2024-11-10T18:13:31,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/a486eefd61e14512be3666cfe5800bd8 2024-11-10T18:13:31,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a23e46a5d4f847b399578f84886ff4e4 is 50, key is test_row_0/C:col10/1731262411408/Put/seqid=0 2024-11-10T18:13:31,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741857_1033 (size=12151) 2024-11-10T18:13:31,551 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:13:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262471555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262471556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262471595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,598 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262471605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,608 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262471774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262471775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:31,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a23e46a5d4f847b399578f84886ff4e4 2024-11-10T18:13:31,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/15b03b5a2a8b48acbf80aca844e45361 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/15b03b5a2a8b48acbf80aca844e45361 2024-11-10T18:13:31,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/15b03b5a2a8b48acbf80aca844e45361, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T18:13:31,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/a486eefd61e14512be3666cfe5800bd8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a486eefd61e14512be3666cfe5800bd8 2024-11-10T18:13:31,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a486eefd61e14512be3666cfe5800bd8, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T18:13:31,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a23e46a5d4f847b399578f84886ff4e4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a23e46a5d4f847b399578f84886ff4e4 2024-11-10T18:13:31,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a23e46a5d4f847b399578f84886ff4e4, entries=150, sequenceid=213, filesize=11.9 K 2024-11-10T18:13:31,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=40.25 KB/41220 for 17351107c8cbd49315660faf09692c94 in 580ms, sequenceid=213, compaction requested=true 2024-11-10T18:13:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:31,991 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:31,991 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:31,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:31,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:31,994 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:31,994 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:31,994 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:31,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-10T18:13:31,995 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f85dce42f3784f1c868d09695339b4f7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/7983f93b692f421ca1aa14196cf3b2ac, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/15b03b5a2a8b48acbf80aca844e45361] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=35.7 K 2024-11-10T18:13:31,995 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:31,995 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:31,995 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:31,995 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ebe23100885e471bb8a6b1580b158358, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b31ef52b87e643cf8074871bdf708ceb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a486eefd61e14512be3666cfe5800bd8] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=35.7 K 2024-11-10T18:13:31,996 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:31,996 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ebe23100885e471bb8a6b1580b158358, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731262409508 2024-11-10T18:13:31,996 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f85dce42f3784f1c868d09695339b4f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731262409508 2024-11-10T18:13:31,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:31,998 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7983f93b692f421ca1aa14196cf3b2ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731262409655 2024-11-10T18:13:31,999 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b31ef52b87e643cf8074871bdf708ceb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731262409655 2024-11-10T18:13:31,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-10T18:13:32,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:32,001 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15b03b5a2a8b48acbf80aca844e45361, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731262411070 2024-11-10T18:13:32,002 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a486eefd61e14512be3666cfe5800bd8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731262411070 2024-11-10T18:13:32,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:32,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:32,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:32,018 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-10T18:13:32,019 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-10T18:13:32,021 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T18:13:32,021 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-10T18:13:32,022 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T18:13:32,022 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T18:13:32,023 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T18:13:32,023 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T18:13:32,036 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#21 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:32,036 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:32,037 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/102ede0288aa450ba9360fd1d420eafb is 50, key is test_row_0/B:col10/1731262411408/Put/seqid=0 2024-11-10T18:13:32,037 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/af08bc0ad95545829b13289a696a4a7f is 50, key is test_row_0/A:col10/1731262411408/Put/seqid=0 2024-11-10T18:13:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741858_1034 (size=12391) 2024-11-10T18:13:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:32,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:13:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:32,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:32,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741859_1035 (size=12391) 2024-11-10T18:13:32,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b305d0911c6d4cdcb1a3c31f39fc23d4 is 50, key is test_row_0/A:col10/1731262412087/Put/seqid=0 2024-11-10T18:13:32,109 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/102ede0288aa450ba9360fd1d420eafb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/102ede0288aa450ba9360fd1d420eafb 2024-11-10T18:13:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:32,122 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into 102ede0288aa450ba9360fd1d420eafb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:32,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:32,122 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=13, startTime=1731262411991; duration=0sec 2024-11-10T18:13:32,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:32,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:32,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:32,128 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36591 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:32,128 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:32,128 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,132 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d196d4a096694c52b567cb5d67350821, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82161791e53242aca78fc678ca7bcccb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a23e46a5d4f847b399578f84886ff4e4] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=35.7 K 2024-11-10T18:13:32,133 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d196d4a096694c52b567cb5d67350821, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1731262409508 2024-11-10T18:13:32,134 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 82161791e53242aca78fc678ca7bcccb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731262409655 2024-11-10T18:13:32,135 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a23e46a5d4f847b399578f84886ff4e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731262411070 2024-11-10T18:13:32,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741860_1036 (size=12151) 2024-11-10T18:13:32,159 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:32,159 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/79aa5fec2b5d4600852b7091d674f519 is 50, key is test_row_0/C:col10/1731262411408/Put/seqid=0 2024-11-10T18:13:32,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,162 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262412145, max timestamp=1731262412182Num uniques 859; 2024-11-10T18:13:32,187 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:32,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262412145, max timestamp=1731262412182Num uniques 859; 2024-11-10T18:13:32,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:32,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262412145, max timestamp=1731262412182Num uniques 859; 2024-11-10T18:13:32,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:32,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741861_1037 (size=12391) 2024-11-10T18:13:32,212 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/79aa5fec2b5d4600852b7091d674f519 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/79aa5fec2b5d4600852b7091d674f519 2024-11-10T18:13:32,226 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 79aa5fec2b5d4600852b7091d674f519(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:32,227 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:32,227 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=13, startTime=1731262411991; duration=0sec 2024-11-10T18:13:32,227 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:32,227 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:32,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262472287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262472288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262472315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:32,320 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:32,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:32,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:32,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262472404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262472405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:32,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,483 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/af08bc0ad95545829b13289a696a4a7f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/af08bc0ad95545829b13289a696a4a7f 2024-11-10T18:13:32,493 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into af08bc0ad95545829b13289a696a4a7f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:32,493 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:32,493 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=13, startTime=1731262411991; duration=0sec 2024-11-10T18:13:32,493 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:32,493 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:32,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b305d0911c6d4cdcb1a3c31f39fc23d4 2024-11-10T18:13:32,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/19343b849a7c4f5e8591a7e0ff50d215 is 50, key is test_row_0/B:col10/1731262412087/Put/seqid=0 2024-11-10T18:13:32,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741862_1038 (size=12151) 2024-11-10T18:13:32,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262472615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262472615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:32,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:32,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:32,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:32,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262472925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262472925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:32,934 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:32,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:32,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:32,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:32,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:32,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/19343b849a7c4f5e8591a7e0ff50d215 2024-11-10T18:13:32,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/15a35ad4b4f54734b346a37586756e58 is 50, key is test_row_0/C:col10/1731262412087/Put/seqid=0 2024-11-10T18:13:32,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741863_1039 (size=12151) 2024-11-10T18:13:33,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:33,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:33,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:33,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:33,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/15a35ad4b4f54734b346a37586756e58 2024-11-10T18:13:33,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b305d0911c6d4cdcb1a3c31f39fc23d4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b305d0911c6d4cdcb1a3c31f39fc23d4 2024-11-10T18:13:33,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b305d0911c6d4cdcb1a3c31f39fc23d4, entries=150, sequenceid=224, filesize=11.9 K 2024-11-10T18:13:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/19343b849a7c4f5e8591a7e0ff50d215 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/19343b849a7c4f5e8591a7e0ff50d215 2024-11-10T18:13:33,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:33,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:33,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,422 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/19343b849a7c4f5e8591a7e0ff50d215, entries=150, sequenceid=224, filesize=11.9 K 2024-11-10T18:13:33,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/15a35ad4b4f54734b346a37586756e58 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/15a35ad4b4f54734b346a37586756e58 2024-11-10T18:13:33,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:33,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262473435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:33,437 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:33,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/15a35ad4b4f54734b346a37586756e58, entries=150, sequenceid=224, filesize=11.9 K 2024-11-10T18:13:33,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 17351107c8cbd49315660faf09692c94 in 1354ms, sequenceid=224, compaction requested=false 2024-11-10T18:13:33,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:33,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:33,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T18:13:33,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:33,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:33,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:33,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:33,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:33,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:33,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/54acc8c738d04a50acaaaa8a85eb9bd2 is 50, key is test_row_0/A:col10/1731262413445/Put/seqid=0 2024-11-10T18:13:33,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741864_1040 (size=14691) 2024-11-10T18:13:33,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262473506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,512 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:33,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262473615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:33,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262473615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:33,619 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:33,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:33,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262473616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:33,620 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:33,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262473824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:33,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:33,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:33,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:33,875 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:33,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/54acc8c738d04a50acaaaa8a85eb9bd2 2024-11-10T18:13:33,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/035b2ccaf736426ebd0c27fc7d8424ff is 50, key is test_row_0/B:col10/1731262413445/Put/seqid=0 2024-11-10T18:13:33,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741865_1041 (size=12301) 2024-11-10T18:13:34,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:34,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:34,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262474135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:34,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:34,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/035b2ccaf736426ebd0c27fc7d8424ff 2024-11-10T18:13:34,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/38d825f018574ce7aa896ce74273498c is 50, key is test_row_0/C:col10/1731262413445/Put/seqid=0 2024-11-10T18:13:34,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741866_1042 (size=12301) 2024-11-10T18:13:34,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/38d825f018574ce7aa896ce74273498c 2024-11-10T18:13:34,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/54acc8c738d04a50acaaaa8a85eb9bd2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/54acc8c738d04a50acaaaa8a85eb9bd2 2024-11-10T18:13:34,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/54acc8c738d04a50acaaaa8a85eb9bd2, entries=200, sequenceid=263, filesize=14.3 K 2024-11-10T18:13:34,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/035b2ccaf736426ebd0c27fc7d8424ff as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/035b2ccaf736426ebd0c27fc7d8424ff 2024-11-10T18:13:34,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/035b2ccaf736426ebd0c27fc7d8424ff, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T18:13:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/38d825f018574ce7aa896ce74273498c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/38d825f018574ce7aa896ce74273498c 2024-11-10T18:13:34,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/38d825f018574ce7aa896ce74273498c, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T18:13:34,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 17351107c8cbd49315660faf09692c94 in 972ms, sequenceid=263, compaction requested=true 2024-11-10T18:13:34,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:34,423 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:34,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:34,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:34,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:34,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:34,425 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:34,428 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:34,428 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:34,428 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,429 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/af08bc0ad95545829b13289a696a4a7f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b305d0911c6d4cdcb1a3c31f39fc23d4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/54acc8c738d04a50acaaaa8a85eb9bd2] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=38.3 K 2024-11-10T18:13:34,432 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting af08bc0ad95545829b13289a696a4a7f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731262411070 2024-11-10T18:13:34,432 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:34,433 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b305d0911c6d4cdcb1a3c31f39fc23d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731262411413 2024-11-10T18:13:34,433 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:34,433 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:34,433 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/102ede0288aa450ba9360fd1d420eafb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/19343b849a7c4f5e8591a7e0ff50d215, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/035b2ccaf736426ebd0c27fc7d8424ff] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.0 K 2024-11-10T18:13:34,433 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54acc8c738d04a50acaaaa8a85eb9bd2, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262412209 2024-11-10T18:13:34,434 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 102ede0288aa450ba9360fd1d420eafb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731262411070 2024-11-10T18:13:34,435 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 19343b849a7c4f5e8591a7e0ff50d215, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731262411413 2024-11-10T18:13:34,435 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 035b2ccaf736426ebd0c27fc7d8424ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262412284 2024-11-10T18:13:34,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:34,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:13:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:34,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:34,463 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:34,464 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c5b26ceca95243a69a28f03c1dcc8699 is 50, key is test_row_0/B:col10/1731262413445/Put/seqid=0 2024-11-10T18:13:34,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/74a86e7208594b0696a9932040cdd03c is 50, key is test_row_0/A:col10/1731262413503/Put/seqid=0 2024-11-10T18:13:34,478 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:34,480 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/e967e7ac019d4d49a8f4dd5f786068d7 is 50, key is test_row_0/A:col10/1731262413445/Put/seqid=0 2024-11-10T18:13:34,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:34,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:34,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741867_1043 (size=12643) 2024-11-10T18:13:34,529 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c5b26ceca95243a69a28f03c1dcc8699 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5b26ceca95243a69a28f03c1dcc8699 2024-11-10T18:13:34,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741869_1045 (size=12643) 2024-11-10T18:13:34,544 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/e967e7ac019d4d49a8f4dd5f786068d7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e967e7ac019d4d49a8f4dd5f786068d7 2024-11-10T18:13:34,547 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into c5b26ceca95243a69a28f03c1dcc8699(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:34,547 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:34,547 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=13, startTime=1731262414425; duration=0sec 2024-11-10T18:13:34,547 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:34,547 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:34,547 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:34,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741868_1044 (size=17181) 2024-11-10T18:13:34,551 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:34,551 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:34,551 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,551 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/79aa5fec2b5d4600852b7091d674f519, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/15a35ad4b4f54734b346a37586756e58, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/38d825f018574ce7aa896ce74273498c] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.0 K 2024-11-10T18:13:34,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/74a86e7208594b0696a9932040cdd03c 2024-11-10T18:13:34,552 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 79aa5fec2b5d4600852b7091d674f519, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731262411070 2024-11-10T18:13:34,554 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 15a35ad4b4f54734b346a37586756e58, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731262411413 2024-11-10T18:13:34,555 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into e967e7ac019d4d49a8f4dd5f786068d7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:34,555 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:34,555 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=13, startTime=1731262414423; duration=0sec 2024-11-10T18:13:34,555 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d825f018574ce7aa896ce74273498c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262412284 2024-11-10T18:13:34,555 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:34,555 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:34,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/568b8c76e32f4843be2ef3f42cac4562 is 50, key is test_row_0/B:col10/1731262413503/Put/seqid=0 2024-11-10T18:13:34,573 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:34,574 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/ead91e33fcb44041884b5d229ec7d0f3 is 50, key is test_row_0/C:col10/1731262413445/Put/seqid=0 2024-11-10T18:13:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262474577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:34,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741870_1046 (size=12301) 2024-11-10T18:13:34,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/568b8c76e32f4843be2ef3f42cac4562 2024-11-10T18:13:34,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741871_1047 (size=12643) 2024-11-10T18:13:34,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/89763a2cc2a0496eaab3c2de1e967738 is 50, key is test_row_0/C:col10/1731262413503/Put/seqid=0 2024-11-10T18:13:34,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741872_1048 (size=12301) 2024-11-10T18:13:34,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/89763a2cc2a0496eaab3c2de1e967738 2024-11-10T18:13:34,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/74a86e7208594b0696a9932040cdd03c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/74a86e7208594b0696a9932040cdd03c 2024-11-10T18:13:34,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262474645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:34,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:34,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:34,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/74a86e7208594b0696a9932040cdd03c, entries=250, sequenceid=275, filesize=16.8 K 2024-11-10T18:13:34,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/568b8c76e32f4843be2ef3f42cac4562 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/568b8c76e32f4843be2ef3f42cac4562 2024-11-10T18:13:34,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/568b8c76e32f4843be2ef3f42cac4562, entries=150, sequenceid=275, filesize=12.0 K 2024-11-10T18:13:34,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/89763a2cc2a0496eaab3c2de1e967738 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/89763a2cc2a0496eaab3c2de1e967738 2024-11-10T18:13:34,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262474685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:34,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/89763a2cc2a0496eaab3c2de1e967738, entries=150, sequenceid=275, filesize=12.0 K 2024-11-10T18:13:34,692 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 17351107c8cbd49315660faf09692c94 in 239ms, sequenceid=275, compaction requested=false 2024-11-10T18:13:34,692 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:34,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-10T18:13:34,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:34,811 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:13:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:34,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/f91104aaf50a4593a1fdfb20da5dc24e is 50, key is test_row_1/A:col10/1731262414558/Put/seqid=0 2024-11-10T18:13:34,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741873_1049 (size=9857) 2024-11-10T18:13:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:34,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:34,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262474931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:34,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:35,004 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/ead91e33fcb44041884b5d229ec7d0f3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/ead91e33fcb44041884b5d229ec7d0f3 2024-11-10T18:13:35,017 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into ead91e33fcb44041884b5d229ec7d0f3(size=12.3 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:35,017 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:35,017 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=13, startTime=1731262414425; duration=0sec 2024-11-10T18:13:35,018 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:35,018 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:35,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262475045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:35,240 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/f91104aaf50a4593a1fdfb20da5dc24e 2024-11-10T18:13:35,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:35,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262475255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:35,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/e66b52e520da4bfa8e8d46fea83a2dc1 is 50, key is test_row_1/B:col10/1731262414558/Put/seqid=0 2024-11-10T18:13:35,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741874_1050 (size=9857) 2024-11-10T18:13:35,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:35,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262475565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:35,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:35,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262475655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:35,660 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:35,679 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/e66b52e520da4bfa8e8d46fea83a2dc1 2024-11-10T18:13:35,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/8e29f921f9d148dfa7f2726b3a052cc1 is 50, key is test_row_1/C:col10/1731262414558/Put/seqid=0 2024-11-10T18:13:35,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741875_1051 (size=9857) 2024-11-10T18:13:35,700 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=297 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/8e29f921f9d148dfa7f2726b3a052cc1 2024-11-10T18:13:35,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/f91104aaf50a4593a1fdfb20da5dc24e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f91104aaf50a4593a1fdfb20da5dc24e 2024-11-10T18:13:35,718 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f91104aaf50a4593a1fdfb20da5dc24e, entries=100, sequenceid=297, filesize=9.6 K 2024-11-10T18:13:35,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/e66b52e520da4bfa8e8d46fea83a2dc1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/e66b52e520da4bfa8e8d46fea83a2dc1 2024-11-10T18:13:35,727 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/e66b52e520da4bfa8e8d46fea83a2dc1, entries=100, sequenceid=297, filesize=9.6 K 2024-11-10T18:13:35,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/8e29f921f9d148dfa7f2726b3a052cc1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8e29f921f9d148dfa7f2726b3a052cc1 2024-11-10T18:13:35,744 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8e29f921f9d148dfa7f2726b3a052cc1, entries=100, sequenceid=297, filesize=9.6 K 2024-11-10T18:13:35,747 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 17351107c8cbd49315660faf09692c94 in 937ms, sequenceid=297, compaction requested=true 2024-11-10T18:13:35,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:35,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:35,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-10T18:13:35,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-10T18:13:35,754 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-10T18:13:35,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7470 sec 2024-11-10T18:13:35,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 3.7590 sec 2024-11-10T18:13:36,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:36,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:13:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:36,081 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:36,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:36,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/ede3ed027c9a46018cd051574aee2c28 is 50, key is test_row_0/A:col10/1731262414925/Put/seqid=0 2024-11-10T18:13:36,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741876_1052 (size=12301) 2024-11-10T18:13:36,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/ede3ed027c9a46018cd051574aee2c28 2024-11-10T18:13:36,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/7add5f14eaca4c62a038c73e62d9944f is 50, key is test_row_0/B:col10/1731262414925/Put/seqid=0 2024-11-10T18:13:36,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741877_1053 (size=12301) 2024-11-10T18:13:36,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/7add5f14eaca4c62a038c73e62d9944f 2024-11-10T18:13:36,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/b10b0ed877d14658b8e00c5be0a76fe9 is 50, key is test_row_0/C:col10/1731262414925/Put/seqid=0 2024-11-10T18:13:36,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741878_1054 (size=12301) 2024-11-10T18:13:36,171 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/b10b0ed877d14658b8e00c5be0a76fe9 2024-11-10T18:13:36,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/ede3ed027c9a46018cd051574aee2c28 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ede3ed027c9a46018cd051574aee2c28 2024-11-10T18:13:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-10T18:13:36,185 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:36,186 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-10T18:13:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T18:13:36,189 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:36,190 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:36,191 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:36,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ede3ed027c9a46018cd051574aee2c28, entries=150, sequenceid=311, filesize=12.0 K 2024-11-10T18:13:36,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/7add5f14eaca4c62a038c73e62d9944f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/7add5f14eaca4c62a038c73e62d9944f 2024-11-10T18:13:36,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/7add5f14eaca4c62a038c73e62d9944f, entries=150, sequenceid=311, filesize=12.0 K 2024-11-10T18:13:36,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262476193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/b10b0ed877d14658b8e00c5be0a76fe9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b10b0ed877d14658b8e00c5be0a76fe9 2024-11-10T18:13:36,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b10b0ed877d14658b8e00c5be0a76fe9, entries=150, sequenceid=311, filesize=12.0 K 2024-11-10T18:13:36,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 17351107c8cbd49315660faf09692c94 in 132ms, sequenceid=311, compaction requested=true 2024-11-10T18:13:36,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:36,212 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:36,213 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:36,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:36,215 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51982 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:36,215 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:36,215 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,215 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e967e7ac019d4d49a8f4dd5f786068d7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/74a86e7208594b0696a9932040cdd03c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f91104aaf50a4593a1fdfb20da5dc24e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ede3ed027c9a46018cd051574aee2c28] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=50.8 K 2024-11-10T18:13:36,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e967e7ac019d4d49a8f4dd5f786068d7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262412284 2024-11-10T18:13:36,217 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47102 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:36,217 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:36,217 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,217 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5b26ceca95243a69a28f03c1dcc8699, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/568b8c76e32f4843be2ef3f42cac4562, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/e66b52e520da4bfa8e8d46fea83a2dc1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/7add5f14eaca4c62a038c73e62d9944f] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=46.0 K 2024-11-10T18:13:36,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74a86e7208594b0696a9932040cdd03c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731262413469 2024-11-10T18:13:36,218 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c5b26ceca95243a69a28f03c1dcc8699, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262412284 2024-11-10T18:13:36,218 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f91104aaf50a4593a1fdfb20da5dc24e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731262414558 2024-11-10T18:13:36,218 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 568b8c76e32f4843be2ef3f42cac4562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731262413494 2024-11-10T18:13:36,218 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede3ed027c9a46018cd051574aee2c28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731262414925 2024-11-10T18:13:36,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e66b52e520da4bfa8e8d46fea83a2dc1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731262414558 2024-11-10T18:13:36,220 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7add5f14eaca4c62a038c73e62d9944f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731262414925 2024-11-10T18:13:36,241 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:36,242 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/f7c374ec0a4b4f35bc41e16d1b7297df is 50, key is test_row_0/B:col10/1731262414925/Put/seqid=0 2024-11-10T18:13:36,243 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:36,244 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/316c83ff01eb44ccadf7e59c3bba5ec1 is 50, key is test_row_0/A:col10/1731262414925/Put/seqid=0 2024-11-10T18:13:36,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741880_1056 (size=12779) 2024-11-10T18:13:36,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741879_1055 (size=12779) 2024-11-10T18:13:36,268 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/316c83ff01eb44ccadf7e59c3bba5ec1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/316c83ff01eb44ccadf7e59c3bba5ec1 2024-11-10T18:13:36,278 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into 316c83ff01eb44ccadf7e59c3bba5ec1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:36,278 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:36,278 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=12, startTime=1731262416212; duration=0sec 2024-11-10T18:13:36,279 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:36,279 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:36,279 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:36,281 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47102 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:36,281 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:36,281 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,281 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/ead91e33fcb44041884b5d229ec7d0f3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/89763a2cc2a0496eaab3c2de1e967738, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8e29f921f9d148dfa7f2726b3a052cc1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b10b0ed877d14658b8e00c5be0a76fe9] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=46.0 K 2024-11-10T18:13:36,282 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead91e33fcb44041884b5d229ec7d0f3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262412284 2024-11-10T18:13:36,282 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89763a2cc2a0496eaab3c2de1e967738, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731262413494 2024-11-10T18:13:36,282 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e29f921f9d148dfa7f2726b3a052cc1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=297, earliestPutTs=1731262414558 2024-11-10T18:13:36,283 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b10b0ed877d14658b8e00c5be0a76fe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731262414925 2024-11-10T18:13:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T18:13:36,296 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#44 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:36,297 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a7a9fe73a4a841a789c25ba47cc654cd is 50, key is test_row_0/C:col10/1731262414925/Put/seqid=0 2024-11-10T18:13:36,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741881_1057 (size=12779) 2024-11-10T18:13:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:36,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:13:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:36,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:36,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/2dd994df19e8441db2b6420b6cc399ee is 50, key is test_row_0/A:col10/1731262416183/Put/seqid=0 2024-11-10T18:13:36,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741882_1058 (size=14741) 2024-11-10T18:13:36,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:36,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262476390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262476390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,397 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T18:13:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262476505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262476505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:36,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:36,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,667 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/f7c374ec0a4b4f35bc41e16d1b7297df as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/f7c374ec0a4b4f35bc41e16d1b7297df 2024-11-10T18:13:36,676 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into f7c374ec0a4b4f35bc41e16d1b7297df(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:36,676 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:36,676 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=12, startTime=1731262416213; duration=0sec 2024-11-10T18:13:36,677 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:36,677 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:36,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262476715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262476715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:36,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:36,719 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a7a9fe73a4a841a789c25ba47cc654cd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a7a9fe73a4a841a789c25ba47cc654cd 2024-11-10T18:13:36,728 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into a7a9fe73a4a841a789c25ba47cc654cd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:36,728 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:36,728 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=12, startTime=1731262416214; duration=0sec 2024-11-10T18:13:36,728 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:36,728 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:36,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/2dd994df19e8441db2b6420b6cc399ee 2024-11-10T18:13:36,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c6cca38b002a4148adf0b36bc85a743d is 50, key is test_row_0/B:col10/1731262416183/Put/seqid=0 2024-11-10T18:13:36,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741883_1059 (size=12301) 2024-11-10T18:13:36,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T18:13:36,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:36,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:36,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:37,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262477025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262477025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:37,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:37,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:37,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:37,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:37,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:37,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c6cca38b002a4148adf0b36bc85a743d 2024-11-10T18:13:37,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/1d83b75f5e5540809289c1559fb4d488 is 50, key is test_row_0/C:col10/1731262416183/Put/seqid=0 2024-11-10T18:13:37,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741884_1060 (size=12301) 2024-11-10T18:13:37,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/1d83b75f5e5540809289c1559fb4d488 2024-11-10T18:13:37,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/2dd994df19e8441db2b6420b6cc399ee as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2dd994df19e8441db2b6420b6cc399ee 2024-11-10T18:13:37,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2dd994df19e8441db2b6420b6cc399ee, entries=200, sequenceid=332, filesize=14.4 K 2024-11-10T18:13:37,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c6cca38b002a4148adf0b36bc85a743d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c6cca38b002a4148adf0b36bc85a743d 2024-11-10T18:13:37,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c6cca38b002a4148adf0b36bc85a743d, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T18:13:37,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/1d83b75f5e5540809289c1559fb4d488 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1d83b75f5e5540809289c1559fb4d488 2024-11-10T18:13:37,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1d83b75f5e5540809289c1559fb4d488, entries=150, sequenceid=332, filesize=12.0 K 2024-11-10T18:13:37,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 17351107c8cbd49315660faf09692c94 in 896ms, sequenceid=332, compaction requested=false 2024-11-10T18:13:37,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:37,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-10T18:13:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:37,271 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:13:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:37,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:37,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:37,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/6bd6dccff1114a439d114161f84fa479 is 50, key is test_row_0/A:col10/1731262416347/Put/seqid=0 2024-11-10T18:13:37,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741885_1061 (size=12301) 2024-11-10T18:13:37,295 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/6bd6dccff1114a439d114161f84fa479 2024-11-10T18:13:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d1c782944c8b48318eae66c5c5fe5ded is 50, key is test_row_0/B:col10/1731262416347/Put/seqid=0 2024-11-10T18:13:37,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741886_1062 (size=12301) 2024-11-10T18:13:37,323 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d1c782944c8b48318eae66c5c5fe5ded 2024-11-10T18:13:37,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T18:13:37,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/895fe839fd0241529302d91c02b3c726 is 50, key is test_row_0/C:col10/1731262416347/Put/seqid=0 2024-11-10T18:13:37,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741887_1063 (size=12301) 2024-11-10T18:13:37,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:37,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:37,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262477574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262477576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262477635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,637 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:37,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262477645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,650 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262477684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262477685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262477695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:37,699 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:37,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:37,748 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/895fe839fd0241529302d91c02b3c726 2024-11-10T18:13:37,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/6bd6dccff1114a439d114161f84fa479 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6bd6dccff1114a439d114161f84fa479 2024-11-10T18:13:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6bd6dccff1114a439d114161f84fa479, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T18:13:37,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d1c782944c8b48318eae66c5c5fe5ded as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1c782944c8b48318eae66c5c5fe5ded 2024-11-10T18:13:37,768 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1c782944c8b48318eae66c5c5fe5ded, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T18:13:37,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/895fe839fd0241529302d91c02b3c726 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/895fe839fd0241529302d91c02b3c726 2024-11-10T18:13:37,777 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/895fe839fd0241529302d91c02b3c726, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T18:13:37,778 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 17351107c8cbd49315660faf09692c94 in 507ms, sequenceid=346, compaction requested=true 2024-11-10T18:13:37,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:37,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:37,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-10T18:13:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-10T18:13:37,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-10T18:13:37,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5890 sec 2024-11-10T18:13:37,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.5970 sec 2024-11-10T18:13:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:37,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T18:13:37,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262417562, max timestamp=1731262417895Num uniques 859; 2024-11-10T18:13:37,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262417562, max timestamp=1731262417895Num uniques 859; 2024-11-10T18:13:37,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262417562, max timestamp=1731262417895Num uniques 859; 2024-11-10T18:13:37,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:37,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:37,900 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:13:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:37,901 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:37,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/ef397b7ba8bb481a89cf88b10547ea25 is 50, key is test_row_0/A:col10/1731262417562/Put/seqid=0 2024-11-10T18:13:37,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741888_1064 (size=17181) 2024-11-10T18:13:37,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262417907, max timestamp=1731262417979Num uniques 859; 2024-11-10T18:13:37,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262417907, max timestamp=1731262417979Num uniques 859; 2024-11-10T18:13:37,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:13:37,983 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:13:37,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262417907, max timestamp=1731262417979Num uniques 859; 2024-11-10T18:13:37,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:13:38,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262478055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262478061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262478164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262478175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/ef397b7ba8bb481a89cf88b10547ea25 2024-11-10T18:13:38,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-10T18:13:38,335 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:38,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:38,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-10T18:13:38,338 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:38,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T18:13:38,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/dca5da101fc94605bed090f8ea248efd is 50, key is test_row_0/B:col10/1731262417562/Put/seqid=0 2024-11-10T18:13:38,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:38,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:38,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741889_1065 (size=12301) 2024-11-10T18:13:38,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/dca5da101fc94605bed090f8ea248efd 2024-11-10T18:13:38,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/9e39509a8b6a4c519fc1681acdc3be01 is 50, key is test_row_0/C:col10/1731262417562/Put/seqid=0 2024-11-10T18:13:38,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741890_1066 (size=12301) 2024-11-10T18:13:38,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262478374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262478385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T18:13:38,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T18:13:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:38,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:38,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:38,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T18:13:38,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:38,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:38,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:38,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T18:13:38,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262478684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:38,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262478695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:38,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:38,762 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/9e39509a8b6a4c519fc1681acdc3be01 2024-11-10T18:13:38,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/ef397b7ba8bb481a89cf88b10547ea25 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ef397b7ba8bb481a89cf88b10547ea25 2024-11-10T18:13:38,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ef397b7ba8bb481a89cf88b10547ea25, entries=250, sequenceid=368, filesize=16.8 K 2024-11-10T18:13:38,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/dca5da101fc94605bed090f8ea248efd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/dca5da101fc94605bed090f8ea248efd 2024-11-10T18:13:38,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/dca5da101fc94605bed090f8ea248efd, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T18:13:38,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/9e39509a8b6a4c519fc1681acdc3be01 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/9e39509a8b6a4c519fc1681acdc3be01 2024-11-10T18:13:38,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/9e39509a8b6a4c519fc1681acdc3be01, entries=150, sequenceid=368, filesize=12.0 K 2024-11-10T18:13:38,791 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 17351107c8cbd49315660faf09692c94 in 893ms, sequenceid=368, compaction requested=true 2024-11-10T18:13:38,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:38,792 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:38,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:38,792 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:38,792 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:38,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:38,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:38,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:38,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:38,793 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57002 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:38,793 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:38,793 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,794 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/316c83ff01eb44ccadf7e59c3bba5ec1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2dd994df19e8441db2b6420b6cc399ee, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6bd6dccff1114a439d114161f84fa479, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ef397b7ba8bb481a89cf88b10547ea25] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=55.7 K 2024-11-10T18:13:38,794 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 316c83ff01eb44ccadf7e59c3bba5ec1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731262414925 2024-11-10T18:13:38,794 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:38,795 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:38,795 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,795 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/f7c374ec0a4b4f35bc41e16d1b7297df, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c6cca38b002a4148adf0b36bc85a743d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1c782944c8b48318eae66c5c5fe5ded, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/dca5da101fc94605bed090f8ea248efd] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=48.5 K 2024-11-10T18:13:38,795 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dd994df19e8441db2b6420b6cc399ee, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731262416166 2024-11-10T18:13:38,795 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f7c374ec0a4b4f35bc41e16d1b7297df, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731262414925 2024-11-10T18:13:38,796 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bd6dccff1114a439d114161f84fa479, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731262416347 2024-11-10T18:13:38,796 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c6cca38b002a4148adf0b36bc85a743d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731262416183 2024-11-10T18:13:38,796 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d1c782944c8b48318eae66c5c5fe5ded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731262416347 2024-11-10T18:13:38,797 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef397b7ba8bb481a89cf88b10547ea25, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262417562 2024-11-10T18:13:38,797 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting dca5da101fc94605bed090f8ea248efd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262417562 2024-11-10T18:13:38,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-10T18:13:38,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:38,799 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-10T18:13:38,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:38,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:38,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b2d263c9bbe2463994521aee7756690f is 50, key is test_row_0/A:col10/1731262418059/Put/seqid=0 2024-11-10T18:13:38,818 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:38,819 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fa25846048ef40ecbb960ad8ed3163c2 is 50, key is test_row_0/B:col10/1731262417562/Put/seqid=0 2024-11-10T18:13:38,824 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#56 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:38,825 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/8e47002ffd6449caa4fcd4ebcadd32a2 is 50, key is test_row_0/A:col10/1731262417562/Put/seqid=0 2024-11-10T18:13:38,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741892_1068 (size=12301) 2024-11-10T18:13:38,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741891_1067 (size=12915) 2024-11-10T18:13:38,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741893_1069 (size=12915) 2024-11-10T18:13:38,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T18:13:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:39,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:39,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262479220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262479221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,227 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,239 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b2d263c9bbe2463994521aee7756690f 2024-11-10T18:13:39,246 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fa25846048ef40ecbb960ad8ed3163c2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fa25846048ef40ecbb960ad8ed3163c2 2024-11-10T18:13:39,250 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/8e47002ffd6449caa4fcd4ebcadd32a2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/8e47002ffd6449caa4fcd4ebcadd32a2 2024-11-10T18:13:39,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/25913795810e4f62bd6d6cf738aef400 is 50, key is test_row_0/B:col10/1731262418059/Put/seqid=0 2024-11-10T18:13:39,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741894_1070 (size=12301) 2024-11-10T18:13:39,257 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into fa25846048ef40ecbb960ad8ed3163c2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:39,257 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,257 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=12, startTime=1731262418792; duration=0sec 2024-11-10T18:13:39,257 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/25913795810e4f62bd6d6cf738aef400 2024-11-10T18:13:39,258 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:39,258 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:39,258 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:39,260 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49682 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:39,260 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:39,260 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,260 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a7a9fe73a4a841a789c25ba47cc654cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1d83b75f5e5540809289c1559fb4d488, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/895fe839fd0241529302d91c02b3c726, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/9e39509a8b6a4c519fc1681acdc3be01] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=48.5 K 2024-11-10T18:13:39,261 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a7a9fe73a4a841a789c25ba47cc654cd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731262414925 2024-11-10T18:13:39,262 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d83b75f5e5540809289c1559fb4d488, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1731262416183 2024-11-10T18:13:39,262 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 895fe839fd0241529302d91c02b3c726, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731262416347 2024-11-10T18:13:39,263 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e39509a8b6a4c519fc1681acdc3be01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262417562 2024-11-10T18:13:39,266 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into 8e47002ffd6449caa4fcd4ebcadd32a2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:39,266 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,266 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=12, startTime=1731262418791; duration=0sec 2024-11-10T18:13:39,266 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:39,266 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:39,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/cea87f9c0d60400ab2f1370278b51654 is 50, key is test_row_0/C:col10/1731262418059/Put/seqid=0 2024-11-10T18:13:39,284 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#59 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:39,285 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/2bfd4c632e864529aadeb51c471a09af is 50, key is test_row_0/C:col10/1731262417562/Put/seqid=0 2024-11-10T18:13:39,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741895_1071 (size=12301) 2024-11-10T18:13:39,293 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/cea87f9c0d60400ab2f1370278b51654 2024-11-10T18:13:39,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741896_1072 (size=12915) 2024-11-10T18:13:39,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b2d263c9bbe2463994521aee7756690f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b2d263c9bbe2463994521aee7756690f 2024-11-10T18:13:39,308 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/2bfd4c632e864529aadeb51c471a09af as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2bfd4c632e864529aadeb51c471a09af 2024-11-10T18:13:39,311 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b2d263c9bbe2463994521aee7756690f, entries=150, sequenceid=405, filesize=12.0 K 2024-11-10T18:13:39,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/25913795810e4f62bd6d6cf738aef400 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/25913795810e4f62bd6d6cf738aef400 2024-11-10T18:13:39,320 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/25913795810e4f62bd6d6cf738aef400, entries=150, sequenceid=405, filesize=12.0 K 2024-11-10T18:13:39,321 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 2bfd4c632e864529aadeb51c471a09af(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:39,321 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,321 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=12, startTime=1731262418793; duration=0sec 2024-11-10T18:13:39,321 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:39,321 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:39,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/cea87f9c0d60400ab2f1370278b51654 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/cea87f9c0d60400ab2f1370278b51654 2024-11-10T18:13:39,332 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/cea87f9c0d60400ab2f1370278b51654, entries=150, sequenceid=405, filesize=12.0 K 2024-11-10T18:13:39,334 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=33.54 KB/34350 for 17351107c8cbd49315660faf09692c94 in 535ms, sequenceid=405, compaction requested=false 2024-11-10T18:13:39,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-10T18:13:39,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-10T18:13:39,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-10T18:13:39,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 997 msec 2024-11-10T18:13:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:39,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:13:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:39,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:39,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:39,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:39,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.0070 sec 2024-11-10T18:13:39,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b3423a7ec4ed40749e79b739ca73c737 is 50, key is test_row_0/A:col10/1731262419342/Put/seqid=0 2024-11-10T18:13:39,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741897_1073 (size=12301) 2024-11-10T18:13:39,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b3423a7ec4ed40749e79b739ca73c737 2024-11-10T18:13:39,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/a9c105b0b94841b1a2f2a2bd9fff3d97 is 50, key is test_row_0/B:col10/1731262419342/Put/seqid=0 2024-11-10T18:13:39,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741898_1074 (size=12301) 2024-11-10T18:13:39,396 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/a9c105b0b94841b1a2f2a2bd9fff3d97 2024-11-10T18:13:39,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/d1ff8ac0eb7f4f40b06c698cfcc07150 is 50, key is test_row_0/C:col10/1731262419342/Put/seqid=0 2024-11-10T18:13:39,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262479402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262419390, max timestamp=1731262419408Num uniques 859; 2024-11-10T18:13:39,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262419390, max timestamp=1731262419408Num uniques 859; 2024-11-10T18:13:39,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:39,412 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:39,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262419390, max timestamp=1731262419408Num uniques 859; 2024-11-10T18:13:39,413 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:39,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741899_1075 (size=12301) 2024-11-10T18:13:39,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/d1ff8ac0eb7f4f40b06c698cfcc07150 2024-11-10T18:13:39,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b3423a7ec4ed40749e79b739ca73c737 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b3423a7ec4ed40749e79b739ca73c737 2024-11-10T18:13:39,449 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b3423a7ec4ed40749e79b739ca73c737, entries=150, sequenceid=419, filesize=12.0 K 2024-11-10T18:13:39,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/a9c105b0b94841b1a2f2a2bd9fff3d97 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a9c105b0b94841b1a2f2a2bd9fff3d97 2024-11-10T18:13:39,456 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a9c105b0b94841b1a2f2a2bd9fff3d97, entries=150, sequenceid=419, filesize=12.0 K 2024-11-10T18:13:39,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/d1ff8ac0eb7f4f40b06c698cfcc07150 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d1ff8ac0eb7f4f40b06c698cfcc07150 2024-11-10T18:13:39,462 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d1ff8ac0eb7f4f40b06c698cfcc07150, entries=150, sequenceid=419, filesize=12.0 K 2024-11-10T18:13:39,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=93.93 KB/96180 for 17351107c8cbd49315660faf09692c94 in 121ms, sequenceid=419, compaction requested=true 2024-11-10T18:13:39,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,464 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:39,465 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:39,466 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:39,466 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,466 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/8e47002ffd6449caa4fcd4ebcadd32a2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b2d263c9bbe2463994521aee7756690f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b3423a7ec4ed40749e79b739ca73c737] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.6 K 2024-11-10T18:13:39,466 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e47002ffd6449caa4fcd4ebcadd32a2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262417562 2024-11-10T18:13:39,467 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2d263c9bbe2463994521aee7756690f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731262418050 2024-11-10T18:13:39,467 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3423a7ec4ed40749e79b739ca73c737, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731262419207 2024-11-10T18:13:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:39,468 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:39,468 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:39,468 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=100.63 KB heapSize=280.69 KB 2024-11-10T18:13:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:39,469 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:39,470 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:39,470 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:39,470 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,470 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fa25846048ef40ecbb960ad8ed3163c2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/25913795810e4f62bd6d6cf738aef400, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a9c105b0b94841b1a2f2a2bd9fff3d97] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.6 K 2024-11-10T18:13:39,471 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting fa25846048ef40ecbb960ad8ed3163c2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262417562 2024-11-10T18:13:39,471 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 25913795810e4f62bd6d6cf738aef400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731262418050 2024-11-10T18:13:39,472 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c105b0b94841b1a2f2a2bd9fff3d97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731262419207 2024-11-10T18:13:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-10T18:13:39,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:39,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:39,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-10T18:13:39,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T18:13:39,479 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:39,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/24b420197b84475292bd700d3212a662 is 50, key is test_row_0/A:col10/1731262419455/Put/seqid=0 2024-11-10T18:13:39,480 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:39,481 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:39,486 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:39,487 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3fffdb20212848b684b2de465e3720ca is 50, key is test_row_0/A:col10/1731262419342/Put/seqid=0 2024-11-10T18:13:39,494 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:39,495 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/de7b9c233fb64565ab44ca542079de31 is 50, key is test_row_0/B:col10/1731262419342/Put/seqid=0 2024-11-10T18:13:39,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741900_1076 (size=14741) 2024-11-10T18:13:39,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741901_1077 (size=13017) 2024-11-10T18:13:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741902_1078 (size=13017) 2024-11-10T18:13:39,518 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/de7b9c233fb64565ab44ca542079de31 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/de7b9c233fb64565ab44ca542079de31 2024-11-10T18:13:39,526 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into de7b9c233fb64565ab44ca542079de31(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:39,526 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,527 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=13, startTime=1731262419468; duration=0sec 2024-11-10T18:13:39,527 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:39,527 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:39,528 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:39,529 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:39,529 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:39,529 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,529 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2bfd4c632e864529aadeb51c471a09af, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/cea87f9c0d60400ab2f1370278b51654, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d1ff8ac0eb7f4f40b06c698cfcc07150] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.6 K 2024-11-10T18:13:39,530 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bfd4c632e864529aadeb51c471a09af, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262417562 2024-11-10T18:13:39,530 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cea87f9c0d60400ab2f1370278b51654, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731262418050 2024-11-10T18:13:39,531 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ff8ac0eb7f4f40b06c698cfcc07150, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731262419207 2024-11-10T18:13:39,544 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#66 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:39,545 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/673f089165174633bb6ef9567620b349 is 50, key is test_row_0/C:col10/1731262419342/Put/seqid=0 2024-11-10T18:13:39,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262479544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262479547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741903_1079 (size=13017) 2024-11-10T18:13:39,566 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/673f089165174633bb6ef9567620b349 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/673f089165174633bb6ef9567620b349 2024-11-10T18:13:39,575 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 673f089165174633bb6ef9567620b349(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:39,575 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,575 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=13, startTime=1731262419468; duration=0sec 2024-11-10T18:13:39,576 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:39,576 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:39,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T18:13:39,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T18:13:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262479655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262479655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,663 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T18:13:39,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:39,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T18:13:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262479864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262479864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:39,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:39,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/24b420197b84475292bd700d3212a662 2024-11-10T18:13:39,921 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3fffdb20212848b684b2de465e3720ca as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3fffdb20212848b684b2de465e3720ca 2024-11-10T18:13:39,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/792128da9f144df88d2fc2f01fd874c6 is 50, key is test_row_0/B:col10/1731262419455/Put/seqid=0 2024-11-10T18:13:39,928 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into 3fffdb20212848b684b2de465e3720ca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:39,928 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:39,928 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=13, startTime=1731262419464; duration=0sec 2024-11-10T18:13:39,928 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:39,928 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:39,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741904_1080 (size=12301) 2024-11-10T18:13:39,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/792128da9f144df88d2fc2f01fd874c6 2024-11-10T18:13:39,939 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/b74d27ad6f4f4654ad33e7e194c82130 is 50, key is test_row_0/C:col10/1731262419455/Put/seqid=0 2024-11-10T18:13:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741905_1081 (size=12301) 2024-11-10T18:13:39,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T18:13:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:39,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T18:13:40,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:40,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:40,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T18:13:40,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262480175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:40,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262480175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:40,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T18:13:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:40,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:40,345 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/b74d27ad6f4f4654ad33e7e194c82130 2024-11-10T18:13:40,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/24b420197b84475292bd700d3212a662 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/24b420197b84475292bd700d3212a662 2024-11-10T18:13:40,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/24b420197b84475292bd700d3212a662, entries=200, sequenceid=450, filesize=14.4 K 2024-11-10T18:13:40,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/792128da9f144df88d2fc2f01fd874c6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/792128da9f144df88d2fc2f01fd874c6 2024-11-10T18:13:40,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/792128da9f144df88d2fc2f01fd874c6, entries=150, sequenceid=450, filesize=12.0 K 2024-11-10T18:13:40,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/b74d27ad6f4f4654ad33e7e194c82130 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b74d27ad6f4f4654ad33e7e194c82130 2024-11-10T18:13:40,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b74d27ad6f4f4654ad33e7e194c82130, entries=150, sequenceid=450, filesize=12.0 K 2024-11-10T18:13:40,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~280.64 KB/287376, currentSize=80.51 KB/82440 for 17351107c8cbd49315660faf09692c94 in 904ms, sequenceid=450, compaction requested=false 2024-11-10T18:13:40,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:40,405 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-10T18:13:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:40,405 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T18:13:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:40,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:40,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:40,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/1c9d62fa8bfe4fafa25daf3ba510938e is 50, key is test_row_0/A:col10/1731262419544/Put/seqid=0 2024-11-10T18:13:40,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741906_1082 (size=12301) 2024-11-10T18:13:40,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T18:13:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:40,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262480724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:40,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262480726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:40,816 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/1c9d62fa8bfe4fafa25daf3ba510938e 2024-11-10T18:13:40,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/6ebe4c5b1622445da169e93e88bc34db is 50, key is test_row_0/B:col10/1731262419544/Put/seqid=0 2024-11-10T18:13:40,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741907_1083 (size=12301) 2024-11-10T18:13:40,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262480835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:40,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:40,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262480835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:40,837 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:40,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262481044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262481045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,233 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/6ebe4c5b1622445da169e93e88bc34db 2024-11-10T18:13:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a8825bdaaf9e4a59ae7d8820394ef136 is 50, key is test_row_0/C:col10/1731262419544/Put/seqid=0 2024-11-10T18:13:41,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741908_1084 (size=12301) 2024-11-10T18:13:41,250 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=468 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a8825bdaaf9e4a59ae7d8820394ef136 2024-11-10T18:13:41,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/1c9d62fa8bfe4fafa25daf3ba510938e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1c9d62fa8bfe4fafa25daf3ba510938e 2024-11-10T18:13:41,262 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1c9d62fa8bfe4fafa25daf3ba510938e, entries=150, sequenceid=468, filesize=12.0 K 2024-11-10T18:13:41,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/6ebe4c5b1622445da169e93e88bc34db as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6ebe4c5b1622445da169e93e88bc34db 2024-11-10T18:13:41,268 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6ebe4c5b1622445da169e93e88bc34db, entries=150, sequenceid=468, filesize=12.0 K 2024-11-10T18:13:41,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/a8825bdaaf9e4a59ae7d8820394ef136 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a8825bdaaf9e4a59ae7d8820394ef136 2024-11-10T18:13:41,276 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a8825bdaaf9e4a59ae7d8820394ef136, entries=150, sequenceid=468, filesize=12.0 K 2024-11-10T18:13:41,277 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 17351107c8cbd49315660faf09692c94 in 872ms, sequenceid=468, compaction requested=true 2024-11-10T18:13:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-10T18:13:41,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-10T18:13:41,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-10T18:13:41,282 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7980 sec 2024-11-10T18:13:41,284 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8080 sec 2024-11-10T18:13:41,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:41,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T18:13:41,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:41,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:41,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:41,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:41,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:41,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:41,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/6600ce0070224f0eb8ba2bca3c08ff2e is 50, key is test_row_0/A:col10/1731262420714/Put/seqid=0 2024-11-10T18:13:41,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741909_1085 (size=17181) 2024-11-10T18:13:41,376 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/6600ce0070224f0eb8ba2bca3c08ff2e 2024-11-10T18:13:41,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262481383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262481386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/8de49c135f6149c2a344fd8ece92884b is 50, key is test_row_0/B:col10/1731262420714/Put/seqid=0 2024-11-10T18:13:41,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741910_1086 (size=12301) 2024-11-10T18:13:41,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/8de49c135f6149c2a344fd8ece92884b 2024-11-10T18:13:41,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/0dc1ece1f61e4a6c8ac8e5530bab34c2 is 50, key is test_row_0/C:col10/1731262420714/Put/seqid=0 2024-11-10T18:13:41,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741911_1087 (size=12301) 2024-11-10T18:13:41,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262481494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262481494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-10T18:13:41,625 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:41,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-10T18:13:41,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T18:13:41,627 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:41,628 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:41,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:41,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262481704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262481705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40494 deadline: 1731262481705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,709 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:41,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T18:13:41,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T18:13:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:41,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:41,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=487 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/0dc1ece1f61e4a6c8ac8e5530bab34c2 2024-11-10T18:13:41,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/6600ce0070224f0eb8ba2bca3c08ff2e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6600ce0070224f0eb8ba2bca3c08ff2e 2024-11-10T18:13:41,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6600ce0070224f0eb8ba2bca3c08ff2e, entries=250, sequenceid=487, filesize=16.8 K 2024-11-10T18:13:41,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/8de49c135f6149c2a344fd8ece92884b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/8de49c135f6149c2a344fd8ece92884b 2024-11-10T18:13:41,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/8de49c135f6149c2a344fd8ece92884b, entries=150, sequenceid=487, filesize=12.0 K 2024-11-10T18:13:41,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/0dc1ece1f61e4a6c8ac8e5530bab34c2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/0dc1ece1f61e4a6c8ac8e5530bab34c2 2024-11-10T18:13:41,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/0dc1ece1f61e4a6c8ac8e5530bab34c2, entries=150, sequenceid=487, filesize=12.0 K 2024-11-10T18:13:41,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 17351107c8cbd49315660faf09692c94 in 498ms, sequenceid=487, compaction requested=true 2024-11-10T18:13:41,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:41,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:41,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:41,855 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:41,856 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:41,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:41,858 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57240 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:41,858 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:41,858 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,858 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3fffdb20212848b684b2de465e3720ca, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/24b420197b84475292bd700d3212a662, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1c9d62fa8bfe4fafa25daf3ba510938e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6600ce0070224f0eb8ba2bca3c08ff2e] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=55.9 K 2024-11-10T18:13:41,859 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:41,859 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:41,859 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,859 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/de7b9c233fb64565ab44ca542079de31, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/792128da9f144df88d2fc2f01fd874c6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6ebe4c5b1622445da169e93e88bc34db, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/8de49c135f6149c2a344fd8ece92884b] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=48.8 K 2024-11-10T18:13:41,859 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fffdb20212848b684b2de465e3720ca, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731262419207 2024-11-10T18:13:41,860 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting de7b9c233fb64565ab44ca542079de31, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731262419207 2024-11-10T18:13:41,860 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b420197b84475292bd700d3212a662, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731262419437 2024-11-10T18:13:41,861 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 792128da9f144df88d2fc2f01fd874c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731262419455 2024-11-10T18:13:41,861 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9d62fa8bfe4fafa25daf3ba510938e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731262419530 2024-11-10T18:13:41,862 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ebe4c5b1622445da169e93e88bc34db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731262419530 2024-11-10T18:13:41,862 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6600ce0070224f0eb8ba2bca3c08ff2e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731262420714 2024-11-10T18:13:41,863 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 8de49c135f6149c2a344fd8ece92884b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731262420714 2024-11-10T18:13:41,877 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:41,878 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/2b5dad8f165243ecba19b151f5f4b82f is 50, key is test_row_0/A:col10/1731262420714/Put/seqid=0 2024-11-10T18:13:41,879 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:41,880 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b56a879f3f764cec8529e6d48d245133 is 50, key is test_row_0/B:col10/1731262420714/Put/seqid=0 2024-11-10T18:13:41,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741912_1088 (size=13153) 2024-11-10T18:13:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741913_1089 (size=13153) 2024-11-10T18:13:41,907 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b56a879f3f764cec8529e6d48d245133 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b56a879f3f764cec8529e6d48d245133 2024-11-10T18:13:41,913 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into b56a879f3f764cec8529e6d48d245133(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:41,913 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:41,913 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=12, startTime=1731262421856; duration=0sec 2024-11-10T18:13:41,913 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:41,913 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:41,913 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:13:41,915 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:13:41,915 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:41,915 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,915 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/673f089165174633bb6ef9567620b349, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b74d27ad6f4f4654ad33e7e194c82130, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a8825bdaaf9e4a59ae7d8820394ef136, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/0dc1ece1f61e4a6c8ac8e5530bab34c2] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=48.8 K 2024-11-10T18:13:41,916 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 673f089165174633bb6ef9567620b349, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1731262419207 2024-11-10T18:13:41,916 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b74d27ad6f4f4654ad33e7e194c82130, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1731262419455 2024-11-10T18:13:41,916 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a8825bdaaf9e4a59ae7d8820394ef136, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=468, earliestPutTs=1731262419530 2024-11-10T18:13:41,917 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dc1ece1f61e4a6c8ac8e5530bab34c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731262420714 2024-11-10T18:13:41,933 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#77 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:41,934 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/7a793365a66547d28806270d0e1c6245 is 50, key is test_row_0/C:col10/1731262420714/Put/seqid=0 2024-11-10T18:13:41,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:41,935 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:41,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741914_1090 (size=13153) 2024-11-10T18:13:41,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd is 50, key is test_row_0/A:col10/1731262421376/Put/seqid=0 2024-11-10T18:13:41,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T18:13:41,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741915_1091 (size=12301) 2024-11-10T18:13:41,947 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd 2024-11-10T18:13:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/494821ae0a9b4ec4b2140e5d8ff4f93f is 50, key is test_row_0/B:col10/1731262421376/Put/seqid=0 2024-11-10T18:13:41,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741916_1092 (size=12301) 2024-11-10T18:13:41,973 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/494821ae0a9b4ec4b2140e5d8ff4f93f 2024-11-10T18:13:41,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/2160b2d9a8874713866448ac0f81e2b1 is 50, key is test_row_0/C:col10/1731262421376/Put/seqid=0 2024-11-10T18:13:41,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741917_1093 (size=12301) 2024-11-10T18:13:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:42,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:42,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262482059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262482060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262482175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262482175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T18:13:42,302 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/2b5dad8f165243ecba19b151f5f4b82f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2b5dad8f165243ecba19b151f5f4b82f 2024-11-10T18:13:42,308 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into 2b5dad8f165243ecba19b151f5f4b82f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:42,309 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:42,309 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=12, startTime=1731262421855; duration=0sec 2024-11-10T18:13:42,309 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:42,309 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:42,347 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/7a793365a66547d28806270d0e1c6245 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7a793365a66547d28806270d0e1c6245 2024-11-10T18:13:42,353 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 7a793365a66547d28806270d0e1c6245(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:42,354 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:42,354 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=12, startTime=1731262421856; duration=0sec 2024-11-10T18:13:42,354 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:42,354 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:42,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262482385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262482385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,391 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=501 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/2160b2d9a8874713866448ac0f81e2b1 2024-11-10T18:13:42,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd 2024-11-10T18:13:42,402 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd, entries=150, sequenceid=501, filesize=12.0 K 2024-11-10T18:13:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/494821ae0a9b4ec4b2140e5d8ff4f93f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/494821ae0a9b4ec4b2140e5d8ff4f93f 2024-11-10T18:13:42,408 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/494821ae0a9b4ec4b2140e5d8ff4f93f, entries=150, sequenceid=501, filesize=12.0 K 2024-11-10T18:13:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/2160b2d9a8874713866448ac0f81e2b1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2160b2d9a8874713866448ac0f81e2b1 2024-11-10T18:13:42,414 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2160b2d9a8874713866448ac0f81e2b1, entries=150, sequenceid=501, filesize=12.0 K 2024-11-10T18:13:42,415 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 17351107c8cbd49315660faf09692c94 in 480ms, sequenceid=501, compaction requested=false 2024-11-10T18:13:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:42,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-10T18:13:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-10T18:13:42,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-10T18:13:42,420 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 789 msec 2024-11-10T18:13:42,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 796 msec 2024-11-10T18:13:42,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:42,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:13:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:42,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/d7e60c20492d4cd08f2dd57c687882a4 is 50, key is test_row_0/A:col10/1731262422051/Put/seqid=0 2024-11-10T18:13:42,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741918_1094 (size=14741) 2024-11-10T18:13:42,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262482723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262482724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-10T18:13:42,765 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:42,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:42,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-10T18:13:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T18:13:42,768 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:42,769 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:42,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:42,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262482834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262482834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:42,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:42,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T18:13:42,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:42,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:42,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:42,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:42,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:42,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:42,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:43,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262483044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:43,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262483044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,046 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:43,076 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:43,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T18:13:43,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/d7e60c20492d4cd08f2dd57c687882a4 2024-11-10T18:13:43,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c5be076871a64367a595abbdff6ac8b2 is 50, key is test_row_0/B:col10/1731262422051/Put/seqid=0 2024-11-10T18:13:43,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741919_1095 (size=12301) 2024-11-10T18:13:43,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:43,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:43,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262483354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:43,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262483354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:43,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:43,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T18:13:43,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c5be076871a64367a595abbdff6ac8b2 2024-11-10T18:13:43,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/1637344a924d4b67867f64cf40674cb7 is 50, key is test_row_0/C:col10/1731262422051/Put/seqid=0 2024-11-10T18:13:43,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741920_1096 (size=12301) 2024-11-10T18:13:43,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:43,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262483864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:43,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262483864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:43,869 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T18:13:43,959 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=523 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/1637344a924d4b67867f64cf40674cb7 2024-11-10T18:13:43,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/d7e60c20492d4cd08f2dd57c687882a4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7e60c20492d4cd08f2dd57c687882a4 2024-11-10T18:13:43,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7e60c20492d4cd08f2dd57c687882a4, entries=200, sequenceid=523, filesize=14.4 K 2024-11-10T18:13:43,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/c5be076871a64367a595abbdff6ac8b2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5be076871a64367a595abbdff6ac8b2 2024-11-10T18:13:43,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5be076871a64367a595abbdff6ac8b2, entries=150, sequenceid=523, filesize=12.0 K 2024-11-10T18:13:43,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/1637344a924d4b67867f64cf40674cb7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1637344a924d4b67867f64cf40674cb7 2024-11-10T18:13:43,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1637344a924d4b67867f64cf40674cb7, entries=150, sequenceid=523, filesize=12.0 K 2024-11-10T18:13:43,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 17351107c8cbd49315660faf09692c94 in 1286ms, sequenceid=523, compaction requested=true 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:43,985 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:43,985 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:43,987 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:43,987 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:43,987 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:43,987 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:43,987 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,987 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,988 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2b5dad8f165243ecba19b151f5f4b82f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7e60c20492d4cd08f2dd57c687882a4] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=39.3 K 2024-11-10T18:13:43,988 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b56a879f3f764cec8529e6d48d245133, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/494821ae0a9b4ec4b2140e5d8ff4f93f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5be076871a64367a595abbdff6ac8b2] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.9 K 2024-11-10T18:13:43,988 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b5dad8f165243ecba19b151f5f4b82f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731262420714 2024-11-10T18:13:43,988 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b56a879f3f764cec8529e6d48d245133, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731262420714 2024-11-10T18:13:43,989 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 494821ae0a9b4ec4b2140e5d8ff4f93f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731262421376 2024-11-10T18:13:43,989 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8c38cbfdaf04f7eb74ff9a8c0f732bd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731262421376 2024-11-10T18:13:43,989 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c5be076871a64367a595abbdff6ac8b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731262422051 2024-11-10T18:13:43,989 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7e60c20492d4cd08f2dd57c687882a4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731262422046 2024-11-10T18:13:43,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-10T18:13:43,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:43,999 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#84 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:43,999 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:13:44,000 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b221280c050f49638d19567d87d1fd51 is 50, key is test_row_0/B:col10/1731262422051/Put/seqid=0 2024-11-10T18:13:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:44,000 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:44,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:44,001 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b7be4abf476945e28608e1af647e95a8 is 50, key is test_row_0/A:col10/1731262422051/Put/seqid=0 2024-11-10T18:13:44,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741921_1097 (size=13255) 2024-11-10T18:13:44,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741922_1098 (size=13255) 2024-11-10T18:13:44,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/1b7d44742cd04f1986d403f6a9f2bfb7 is 50, key is test_row_0/A:col10/1731262422714/Put/seqid=0 2024-11-10T18:13:44,020 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/b221280c050f49638d19567d87d1fd51 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b221280c050f49638d19567d87d1fd51 2024-11-10T18:13:44,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741923_1099 (size=12301) 2024-11-10T18:13:44,030 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/1b7d44742cd04f1986d403f6a9f2bfb7 2024-11-10T18:13:44,031 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into b221280c050f49638d19567d87d1fd51(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:44,031 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:44,032 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=13, startTime=1731262423985; duration=0sec 2024-11-10T18:13:44,032 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:44,032 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:44,032 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:44,034 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:44,034 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:44,034 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:44,034 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7a793365a66547d28806270d0e1c6245, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2160b2d9a8874713866448ac0f81e2b1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1637344a924d4b67867f64cf40674cb7] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=36.9 K 2024-11-10T18:13:44,034 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a793365a66547d28806270d0e1c6245, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=487, earliestPutTs=1731262420714 2024-11-10T18:13:44,036 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 2160b2d9a8874713866448ac0f81e2b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=501, earliestPutTs=1731262421376 2024-11-10T18:13:44,037 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1637344a924d4b67867f64cf40674cb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731262422051 2024-11-10T18:13:44,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/ac3ca741a7c8491d8b44628748ef0303 is 50, key is test_row_0/B:col10/1731262422714/Put/seqid=0 2024-11-10T18:13:44,051 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:44,052 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/40294d96b0d44444b145039e4ef824b4 is 50, key is test_row_0/C:col10/1731262422051/Put/seqid=0 2024-11-10T18:13:44,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741924_1100 (size=12301) 2024-11-10T18:13:44,065 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/ac3ca741a7c8491d8b44628748ef0303 2024-11-10T18:13:44,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741925_1101 (size=13255) 2024-11-10T18:13:44,076 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/40294d96b0d44444b145039e4ef824b4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/40294d96b0d44444b145039e4ef824b4 2024-11-10T18:13:44,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/c7d910dc5a6142329750e05030af730a is 50, key is test_row_0/C:col10/1731262422714/Put/seqid=0 2024-11-10T18:13:44,085 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 40294d96b0d44444b145039e4ef824b4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:44,085 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:44,085 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=13, startTime=1731262423985; duration=0sec 2024-11-10T18:13:44,085 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:44,085 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:44,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741926_1102 (size=12301) 2024-11-10T18:13:44,420 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/b7be4abf476945e28608e1af647e95a8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b7be4abf476945e28608e1af647e95a8 2024-11-10T18:13:44,427 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into b7be4abf476945e28608e1af647e95a8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:44,427 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:44,427 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=13, startTime=1731262423985; duration=0sec 2024-11-10T18:13:44,427 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:44,427 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:44,488 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/c7d910dc5a6142329750e05030af730a 2024-11-10T18:13:44,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/1b7d44742cd04f1986d403f6a9f2bfb7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1b7d44742cd04f1986d403f6a9f2bfb7 2024-11-10T18:13:44,504 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1b7d44742cd04f1986d403f6a9f2bfb7, entries=150, sequenceid=535, filesize=12.0 K 2024-11-10T18:13:44,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/ac3ca741a7c8491d8b44628748ef0303 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ac3ca741a7c8491d8b44628748ef0303 2024-11-10T18:13:44,513 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ac3ca741a7c8491d8b44628748ef0303, entries=150, sequenceid=535, filesize=12.0 K 2024-11-10T18:13:44,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/c7d910dc5a6142329750e05030af730a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/c7d910dc5a6142329750e05030af730a 2024-11-10T18:13:44,523 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/c7d910dc5a6142329750e05030af730a, entries=150, sequenceid=535, filesize=12.0 K 2024-11-10T18:13:44,524 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 17351107c8cbd49315660faf09692c94 in 525ms, sequenceid=535, compaction requested=false 2024-11-10T18:13:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:44,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-10T18:13:44,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-10T18:13:44,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-10T18:13:44,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7570 sec 2024-11-10T18:13:44,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7630 sec 2024-11-10T18:13:44,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:44,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:13:44,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:44,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:44,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:44,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:44,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:44,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:44,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3f1bbe2e3a8646b494a7dc974b0671c4 is 50, key is test_row_0/A:col10/1731262424891/Put/seqid=0 2024-11-10T18:13:44,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741927_1103 (size=14741) 2024-11-10T18:13:44,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-10T18:13:44,925 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:44,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:13:44,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-10T18:13:44,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:44,927 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:13:44,928 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:13:44,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:13:44,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262424909, max timestamp=1731262424929Num uniques 859; 2024-11-10T18:13:44,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262424909, max timestamp=1731262424929Num uniques 859; 2024-11-10T18:13:44,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:44,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:44,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262424909, max timestamp=1731262424929Num uniques 859; 2024-11-10T18:13:44,933 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:13:44,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:44,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262484976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:44,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:44,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:44,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262484976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:44,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:45,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:45,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:45,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262485085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262485094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,098 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:45,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:45,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262485294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,300 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262485304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,308 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3f1bbe2e3a8646b494a7dc974b0671c4 2024-11-10T18:13:45,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fc36fe32cd7944eb8d6af2bcfe150086 is 50, key is test_row_0/B:col10/1731262424891/Put/seqid=0 2024-11-10T18:13:45,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741928_1104 (size=12301) 2024-11-10T18:13:45,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:45,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,388 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:45,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262485605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,608 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262485614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:45,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:45,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,722 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fc36fe32cd7944eb8d6af2bcfe150086 2024-11-10T18:13:45,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/eada90e34f534ff1909fad051ae10572 is 50, key is test_row_0/C:col10/1731262424891/Put/seqid=0 2024-11-10T18:13:45,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741929_1105 (size=12301) 2024-11-10T18:13:45,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:45,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:45,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,001 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:46,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:46,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:46,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262486115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:46,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:46,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262486124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:46,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:46,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/eada90e34f534ff1909fad051ae10572 2024-11-10T18:13:46,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3f1bbe2e3a8646b494a7dc974b0671c4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3f1bbe2e3a8646b494a7dc974b0671c4 2024-11-10T18:13:46,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3f1bbe2e3a8646b494a7dc974b0671c4, entries=200, sequenceid=548, filesize=14.4 K 2024-11-10T18:13:46,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fc36fe32cd7944eb8d6af2bcfe150086 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fc36fe32cd7944eb8d6af2bcfe150086 2024-11-10T18:13:46,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fc36fe32cd7944eb8d6af2bcfe150086, entries=150, sequenceid=548, filesize=12.0 K 2024-11-10T18:13:46,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/eada90e34f534ff1909fad051ae10572 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/eada90e34f534ff1909fad051ae10572 2024-11-10T18:13:46,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:13:46,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/eada90e34f534ff1909fad051ae10572, entries=150, sequenceid=548, filesize=12.0 K 2024-11-10T18:13:46,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 17351107c8cbd49315660faf09692c94 in 1272ms, sequenceid=548, compaction requested=true 2024-11-10T18:13:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:46,164 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:46,164 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:46,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:46,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:46,166 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:46,166 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40297 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:46,166 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:46,166 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:46,166 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,166 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,166 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b221280c050f49638d19567d87d1fd51, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ac3ca741a7c8491d8b44628748ef0303, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fc36fe32cd7944eb8d6af2bcfe150086] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=37.0 K 2024-11-10T18:13:46,166 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b7be4abf476945e28608e1af647e95a8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1b7d44742cd04f1986d403f6a9f2bfb7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3f1bbe2e3a8646b494a7dc974b0671c4] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=39.4 K 2024-11-10T18:13:46,168 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b221280c050f49638d19567d87d1fd51, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731262422051 2024-11-10T18:13:46,168 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b7be4abf476945e28608e1af647e95a8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731262422051 2024-11-10T18:13:46,168 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac3ca741a7c8491d8b44628748ef0303, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731262422714 2024-11-10T18:13:46,169 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b7d44742cd04f1986d403f6a9f2bfb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731262422714 2024-11-10T18:13:46,169 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc36fe32cd7944eb8d6af2bcfe150086, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731262424887 2024-11-10T18:13:46,169 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f1bbe2e3a8646b494a7dc974b0671c4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731262424887 2024-11-10T18:13:46,180 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:46,181 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:46,181 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/d7260aa3ce6b4454a9f510ab7f486f00 is 50, key is test_row_0/A:col10/1731262424891/Put/seqid=0 2024-11-10T18:13:46,182 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/6919802c0c41411fb1a5d433d72c4dc0 is 50, key is test_row_0/B:col10/1731262424891/Put/seqid=0 2024-11-10T18:13:46,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741931_1107 (size=13357) 2024-11-10T18:13:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741930_1106 (size=13357) 2024-11-10T18:13:46,209 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/d7260aa3ce6b4454a9f510ab7f486f00 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7260aa3ce6b4454a9f510ab7f486f00 2024-11-10T18:13:46,218 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into d7260aa3ce6b4454a9f510ab7f486f00(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:46,218 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:46,219 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=13, startTime=1731262426164; duration=0sec 2024-11-10T18:13:46,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:46,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:46,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:46,221 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:46,221 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:46,221 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,221 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/40294d96b0d44444b145039e4ef824b4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/c7d910dc5a6142329750e05030af730a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/eada90e34f534ff1909fad051ae10572] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=37.0 K 2024-11-10T18:13:46,222 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 40294d96b0d44444b145039e4ef824b4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=523, earliestPutTs=1731262422051 2024-11-10T18:13:46,222 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d910dc5a6142329750e05030af730a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731262422714 2024-11-10T18:13:46,223 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting eada90e34f534ff1909fad051ae10572, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731262424887 2024-11-10T18:13:46,231 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:46,231 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/62fc4b05e1084b989ec276c64c193617 is 50, key is test_row_0/C:col10/1731262424891/Put/seqid=0 2024-11-10T18:13:46,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741932_1108 (size=13357) 2024-11-10T18:13:46,244 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/62fc4b05e1084b989ec276c64c193617 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/62fc4b05e1084b989ec276c64c193617 2024-11-10T18:13:46,249 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 62fc4b05e1084b989ec276c64c193617(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:46,250 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:46,250 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=13, startTime=1731262426164; duration=0sec 2024-11-10T18:13:46,250 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:46,250 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:46,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-10T18:13:46,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:46,308 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:46,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:46,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/20c636898cb6427590467f99ca8fd949 is 50, key is test_row_0/A:col10/1731262424973/Put/seqid=0 2024-11-10T18:13:46,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741933_1109 (size=12301) 2024-11-10T18:13:46,600 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/6919802c0c41411fb1a5d433d72c4dc0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6919802c0c41411fb1a5d433d72c4dc0 2024-11-10T18:13:46,611 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into 6919802c0c41411fb1a5d433d72c4dc0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:46,612 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:46,612 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=13, startTime=1731262426164; duration=0sec 2024-11-10T18:13:46,612 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:46,612 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:46,722 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/20c636898cb6427590467f99ca8fd949 2024-11-10T18:13:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d242186d02cc41c5b1d6ccd08aafd916 is 50, key is test_row_0/B:col10/1731262424973/Put/seqid=0 2024-11-10T18:13:46,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741934_1110 (size=12301) 2024-11-10T18:13:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:47,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. as already flushing 2024-11-10T18:13:47,147 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d242186d02cc41c5b1d6ccd08aafd916 2024-11-10T18:13:47,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262487144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262487145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/e341edb8e36a488c8fa9b6ef8b0066c4 is 50, key is test_row_0/C:col10/1731262424973/Put/seqid=0 2024-11-10T18:13:47,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741935_1111 (size=12301) 2024-11-10T18:13:47,171 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=586 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/e341edb8e36a488c8fa9b6ef8b0066c4 2024-11-10T18:13:47,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/20c636898cb6427590467f99ca8fd949 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/20c636898cb6427590467f99ca8fd949 2024-11-10T18:13:47,182 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/20c636898cb6427590467f99ca8fd949, entries=150, sequenceid=586, filesize=12.0 K 2024-11-10T18:13:47,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d242186d02cc41c5b1d6ccd08aafd916 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d242186d02cc41c5b1d6ccd08aafd916 2024-11-10T18:13:47,188 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d242186d02cc41c5b1d6ccd08aafd916, entries=150, sequenceid=586, filesize=12.0 K 2024-11-10T18:13:47,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/e341edb8e36a488c8fa9b6ef8b0066c4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e341edb8e36a488c8fa9b6ef8b0066c4 2024-11-10T18:13:47,194 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e341edb8e36a488c8fa9b6ef8b0066c4, entries=150, sequenceid=586, filesize=12.0 K 2024-11-10T18:13:47,195 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 17351107c8cbd49315660faf09692c94 in 887ms, sequenceid=586, compaction requested=false 2024-11-10T18:13:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-10T18:13:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-10T18:13:47,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-10T18:13:47,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2690 sec 2024-11-10T18:13:47,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.2740 sec 2024-11-10T18:13:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:47,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:13:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:47,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:47,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:47,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/22e9a2fae7de4008bc37f43e0fbe6520 is 50, key is test_row_0/A:col10/1731262427125/Put/seqid=0 2024-11-10T18:13:47,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741936_1112 (size=17181) 2024-11-10T18:13:47,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/22e9a2fae7de4008bc37f43e0fbe6520 2024-11-10T18:13:47,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/32bb538503d3428ca508889af9446319 is 50, key is test_row_0/B:col10/1731262427125/Put/seqid=0 2024-11-10T18:13:47,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262487294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741937_1113 (size=12301) 2024-11-10T18:13:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262487298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,303 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262487405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262487405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262487615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262487615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,670 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40444 deadline: 1731262487664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,671 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-10T18:13:47,671 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:47,671 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,673 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-10T18:13:47,673 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:47,674 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-10T18:13:47,674 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:47,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,675 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-10T18:13:47,675 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40490 deadline: 1731262487674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,675 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 17351107c8cbd49315660faf09692c94 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:13:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,677 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:47,677 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:47,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:47,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:47,677 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:47,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/32bb538503d3428ca508889af9446319 2024-11-10T18:13:47,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/e4208f0add0741c29cbd60eaa35df22c is 50, key is test_row_0/C:col10/1731262427125/Put/seqid=0 2024-11-10T18:13:47,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741938_1114 (size=12301) 2024-11-10T18:13:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:13:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 295 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40488 deadline: 1731262487924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40500 deadline: 1731262487924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 2024-11-10T18:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:13:48,112 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=599 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/e4208f0add0741c29cbd60eaa35df22c 2024-11-10T18:13:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/22e9a2fae7de4008bc37f43e0fbe6520 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/22e9a2fae7de4008bc37f43e0fbe6520 2024-11-10T18:13:48,121 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/22e9a2fae7de4008bc37f43e0fbe6520, entries=250, sequenceid=599, filesize=16.8 K 2024-11-10T18:13:48,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/32bb538503d3428ca508889af9446319 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/32bb538503d3428ca508889af9446319 2024-11-10T18:13:48,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/32bb538503d3428ca508889af9446319, entries=150, sequenceid=599, filesize=12.0 K 2024-11-10T18:13:48,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/e4208f0add0741c29cbd60eaa35df22c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e4208f0add0741c29cbd60eaa35df22c 2024-11-10T18:13:48,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e4208f0add0741c29cbd60eaa35df22c, entries=150, sequenceid=599, filesize=12.0 K 2024-11-10T18:13:48,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 17351107c8cbd49315660faf09692c94 in 875ms, sequenceid=599, compaction requested=true 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:48,132 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 17351107c8cbd49315660faf09692c94:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:13:48,132 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:48,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:48,133 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:48,133 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:48,133 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/A is initiating minor compaction (all files) 2024-11-10T18:13:48,133 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/B is initiating minor compaction (all files) 2024-11-10T18:13:48,133 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/A in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:48,133 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/B in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:48,133 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7260aa3ce6b4454a9f510ab7f486f00, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/20c636898cb6427590467f99ca8fd949, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/22e9a2fae7de4008bc37f43e0fbe6520] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=41.8 K 2024-11-10T18:13:48,133 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6919802c0c41411fb1a5d433d72c4dc0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d242186d02cc41c5b1d6ccd08aafd916, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/32bb538503d3428ca508889af9446319] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=37.1 K 2024-11-10T18:13:48,134 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7260aa3ce6b4454a9f510ab7f486f00, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731262424887 2024-11-10T18:13:48,134 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6919802c0c41411fb1a5d433d72c4dc0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731262424887 2024-11-10T18:13:48,134 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20c636898cb6427590467f99ca8fd949, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731262424956 2024-11-10T18:13:48,134 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d242186d02cc41c5b1d6ccd08aafd916, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731262424956 2024-11-10T18:13:48,134 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22e9a2fae7de4008bc37f43e0fbe6520, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731262427125 2024-11-10T18:13:48,134 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 32bb538503d3428ca508889af9446319, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731262427125 2024-11-10T18:13:48,144 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#B#compaction#102 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:48,144 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d1f4eb176754458296bd2d414bd2d7da is 50, key is test_row_0/B:col10/1731262427125/Put/seqid=0 2024-11-10T18:13:48,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741939_1115 (size=13459) 2024-11-10T18:13:48,169 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#A#compaction#103 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:48,171 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/cb65e38ef5b141ac90b59c1860b9c605 is 50, key is test_row_0/A:col10/1731262427125/Put/seqid=0 2024-11-10T18:13:48,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741940_1116 (size=13459) 2024-11-10T18:13:48,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:48,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-10T18:13:48,437 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-10T18:13:48,437 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:48,437 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:48,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:48,438 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:48,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:13:48,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:48,438 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-10T18:13:48,438 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:48,438 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:48,438 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:48,439 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:48,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3efd40ac4e9b42d89365861879d859dc is 50, key is test_row_0/A:col10/1731262428435/Put/seqid=0 2024-11-10T18:13:48,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741941_1117 (size=12301) 2024-11-10T18:13:48,556 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d1f4eb176754458296bd2d414bd2d7da as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1f4eb176754458296bd2d414bd2d7da 2024-11-10T18:13:48,563 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/B of 17351107c8cbd49315660faf09692c94 into d1f4eb176754458296bd2d414bd2d7da(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:48,563 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:48,563 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/B, priority=13, startTime=1731262428132; duration=0sec 2024-11-10T18:13:48,563 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:13:48,563 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:B 2024-11-10T18:13:48,563 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:13:48,564 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:13:48,564 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 17351107c8cbd49315660faf09692c94/C is initiating minor compaction (all files) 2024-11-10T18:13:48,564 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 17351107c8cbd49315660faf09692c94/C in TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:48,564 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/62fc4b05e1084b989ec276c64c193617, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e341edb8e36a488c8fa9b6ef8b0066c4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e4208f0add0741c29cbd60eaa35df22c] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp, totalSize=37.1 K 2024-11-10T18:13:48,565 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 62fc4b05e1084b989ec276c64c193617, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1731262424887 2024-11-10T18:13:48,565 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e341edb8e36a488c8fa9b6ef8b0066c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=586, earliestPutTs=1731262424956 2024-11-10T18:13:48,566 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e4208f0add0741c29cbd60eaa35df22c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=599, earliestPutTs=1731262427125 2024-11-10T18:13:48,576 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 17351107c8cbd49315660faf09692c94#C#compaction#105 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:13:48,577 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/4895e96195d64b9dadcd8ff1d9e100ae is 50, key is test_row_0/C:col10/1731262427125/Put/seqid=0 2024-11-10T18:13:48,581 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/cb65e38ef5b141ac90b59c1860b9c605 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/cb65e38ef5b141ac90b59c1860b9c605 2024-11-10T18:13:48,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741942_1118 (size=13459) 2024-11-10T18:13:48,586 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/A of 17351107c8cbd49315660faf09692c94 into cb65e38ef5b141ac90b59c1860b9c605(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:48,586 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:48,586 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/A, priority=13, startTime=1731262428132; duration=0sec 2024-11-10T18:13:48,586 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:48,586 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:A 2024-11-10T18:13:48,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3efd40ac4e9b42d89365861879d859dc 2024-11-10T18:13:48,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d6415e491ae24fef9fc5ddfb4c5003c1 is 50, key is test_row_0/B:col10/1731262428435/Put/seqid=0 2024-11-10T18:13:48,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741943_1119 (size=12301) 2024-11-10T18:13:48,987 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/4895e96195d64b9dadcd8ff1d9e100ae as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/4895e96195d64b9dadcd8ff1d9e100ae 2024-11-10T18:13:48,993 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 17351107c8cbd49315660faf09692c94/C of 17351107c8cbd49315660faf09692c94 into 4895e96195d64b9dadcd8ff1d9e100ae(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:13:48,993 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:48,993 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., storeName=17351107c8cbd49315660faf09692c94/C, priority=13, startTime=1731262428132; duration=0sec 2024-11-10T18:13:48,993 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:13:48,993 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 17351107c8cbd49315660faf09692c94:C 2024-11-10T18:13:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-10T18:13:49,106 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:49,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d6415e491ae24fef9fc5ddfb4c5003c1 2024-11-10T18:13:49,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/7361cdbeb7f840c4a0299c1e9e0fdb28 is 50, key is test_row_0/C:col10/1731262428435/Put/seqid=0 2024-11-10T18:13:49,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741944_1120 (size=12301) 2024-11-10T18:13:49,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=621 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/7361cdbeb7f840c4a0299c1e9e0fdb28 2024-11-10T18:13:49,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/3efd40ac4e9b42d89365861879d859dc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3efd40ac4e9b42d89365861879d859dc 2024-11-10T18:13:49,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3efd40ac4e9b42d89365861879d859dc, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T18:13:49,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/d6415e491ae24fef9fc5ddfb4c5003c1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d6415e491ae24fef9fc5ddfb4c5003c1 2024-11-10T18:13:49,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d6415e491ae24fef9fc5ddfb4c5003c1, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T18:13:49,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/7361cdbeb7f840c4a0299c1e9e0fdb28 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7361cdbeb7f840c4a0299c1e9e0fdb28 2024-11-10T18:13:49,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7361cdbeb7f840c4a0299c1e9e0fdb28, entries=150, sequenceid=621, filesize=12.0 K 2024-11-10T18:13:49,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=0 B/0 for 17351107c8cbd49315660faf09692c94 in 1257ms, sequenceid=621, compaction requested=false 2024-11-10T18:13:49,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 17351107c8cbd49315660faf09692c94: 2024-11-10T18:13:50,492 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T18:13:51,796 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-10T18:13:51,796 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:51,796 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:51,796 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:51,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:57,727 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-10T18:13:57,727 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-10T18:13:57,727 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:57,727 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:13:57,727 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:57,727 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:57,727 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:57,727 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:57,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:57,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 133 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 230 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2959 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2879 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1257 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3766 rows 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1256 2024-11-10T18:13:57,728 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3757 rows 2024-11-10T18:13:57,729 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:13:57,729 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:13:57,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:57,729 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:13:57,729 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:13:57,733 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T18:13:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:57,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T18:13:57,741 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262437740"}]},"ts":"1731262437740"} 2024-11-10T18:13:57,743 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T18:13:57,743 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T18:13:57,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:13:57,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, UNASSIGN}] 2024-11-10T18:13:57,749 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, UNASSIGN 2024-11-10T18:13:57,750 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=17351107c8cbd49315660faf09692c94, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:13:57,752 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, UNASSIGN because future has completed 2024-11-10T18:13:57,752 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:13:57,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:13:57,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T18:13:57,909 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:57,909 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:13:57,909 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 17351107c8cbd49315660faf09692c94, disabling compactions & flushes 2024-11-10T18:13:57,910 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. after waiting 0 ms 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:57,910 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 17351107c8cbd49315660faf09692c94 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=A 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=B 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 17351107c8cbd49315660faf09692c94, store=C 2024-11-10T18:13:57,910 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:13:57,914 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/c02463156698482998dda59397301dd3 is 50, key is test_row_0/A:col10/1731262431795/Put/seqid=0 2024-11-10T18:13:57,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741945_1121 (size=12301) 2024-11-10T18:13:58,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T18:13:58,319 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/c02463156698482998dda59397301dd3 2024-11-10T18:13:58,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fd2936665d26449e8171f8f83744c150 is 50, key is test_row_0/B:col10/1731262431795/Put/seqid=0 2024-11-10T18:13:58,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741946_1122 (size=12301) 2024-11-10T18:13:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T18:13:58,731 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fd2936665d26449e8171f8f83744c150 2024-11-10T18:13:58,739 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/82153fb9bfe743a6a0524a7540b20c23 is 50, key is test_row_0/C:col10/1731262431795/Put/seqid=0 2024-11-10T18:13:58,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741947_1123 (size=12301) 2024-11-10T18:13:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T18:13:59,144 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=630 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/82153fb9bfe743a6a0524a7540b20c23 2024-11-10T18:13:59,150 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/A/c02463156698482998dda59397301dd3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/c02463156698482998dda59397301dd3 2024-11-10T18:13:59,155 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/c02463156698482998dda59397301dd3, entries=150, sequenceid=630, filesize=12.0 K 2024-11-10T18:13:59,156 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/B/fd2936665d26449e8171f8f83744c150 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fd2936665d26449e8171f8f83744c150 2024-11-10T18:13:59,160 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fd2936665d26449e8171f8f83744c150, entries=150, sequenceid=630, filesize=12.0 K 2024-11-10T18:13:59,161 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/.tmp/C/82153fb9bfe743a6a0524a7540b20c23 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82153fb9bfe743a6a0524a7540b20c23 2024-11-10T18:13:59,166 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82153fb9bfe743a6a0524a7540b20c23, entries=150, sequenceid=630, filesize=12.0 K 2024-11-10T18:13:59,167 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 17351107c8cbd49315660faf09692c94 in 1257ms, sequenceid=630, compaction requested=true 2024-11-10T18:13:59,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/5e094b6875c141d48747ffa8d2ffa922, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/a9e06b171a0041008ea276dd1887c1c6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/16975c232c0b474b8684272d25352890, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f85dce42f3784f1c868d09695339b4f7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3c04f9f74e7d493a9c2cfb32156d8461, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/7983f93b692f421ca1aa14196cf3b2ac, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/af08bc0ad95545829b13289a696a4a7f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/15b03b5a2a8b48acbf80aca844e45361, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b305d0911c6d4cdcb1a3c31f39fc23d4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/54acc8c738d04a50acaaaa8a85eb9bd2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e967e7ac019d4d49a8f4dd5f786068d7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/74a86e7208594b0696a9932040cdd03c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f91104aaf50a4593a1fdfb20da5dc24e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/316c83ff01eb44ccadf7e59c3bba5ec1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ede3ed027c9a46018cd051574aee2c28, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2dd994df19e8441db2b6420b6cc399ee, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6bd6dccff1114a439d114161f84fa479, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ef397b7ba8bb481a89cf88b10547ea25, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/8e47002ffd6449caa4fcd4ebcadd32a2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b2d263c9bbe2463994521aee7756690f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3fffdb20212848b684b2de465e3720ca, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b3423a7ec4ed40749e79b739ca73c737, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/24b420197b84475292bd700d3212a662, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1c9d62fa8bfe4fafa25daf3ba510938e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6600ce0070224f0eb8ba2bca3c08ff2e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2b5dad8f165243ecba19b151f5f4b82f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7e60c20492d4cd08f2dd57c687882a4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b7be4abf476945e28608e1af647e95a8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1b7d44742cd04f1986d403f6a9f2bfb7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3f1bbe2e3a8646b494a7dc974b0671c4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7260aa3ce6b4454a9f510ab7f486f00, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/20c636898cb6427590467f99ca8fd949, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/22e9a2fae7de4008bc37f43e0fbe6520] to archive 2024-11-10T18:13:59,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:13:59,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/5e094b6875c141d48747ffa8d2ffa922 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/5e094b6875c141d48747ffa8d2ffa922 2024-11-10T18:13:59,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/a9e06b171a0041008ea276dd1887c1c6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/a9e06b171a0041008ea276dd1887c1c6 2024-11-10T18:13:59,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/16975c232c0b474b8684272d25352890 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/16975c232c0b474b8684272d25352890 2024-11-10T18:13:59,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f85dce42f3784f1c868d09695339b4f7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f85dce42f3784f1c868d09695339b4f7 2024-11-10T18:13:59,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3c04f9f74e7d493a9c2cfb32156d8461 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3c04f9f74e7d493a9c2cfb32156d8461 2024-11-10T18:13:59,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/7983f93b692f421ca1aa14196cf3b2ac to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/7983f93b692f421ca1aa14196cf3b2ac 2024-11-10T18:13:59,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/af08bc0ad95545829b13289a696a4a7f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/af08bc0ad95545829b13289a696a4a7f 2024-11-10T18:13:59,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/15b03b5a2a8b48acbf80aca844e45361 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/15b03b5a2a8b48acbf80aca844e45361 2024-11-10T18:13:59,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b305d0911c6d4cdcb1a3c31f39fc23d4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b305d0911c6d4cdcb1a3c31f39fc23d4 2024-11-10T18:13:59,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/54acc8c738d04a50acaaaa8a85eb9bd2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/54acc8c738d04a50acaaaa8a85eb9bd2 2024-11-10T18:13:59,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e967e7ac019d4d49a8f4dd5f786068d7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e967e7ac019d4d49a8f4dd5f786068d7 2024-11-10T18:13:59,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/74a86e7208594b0696a9932040cdd03c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/74a86e7208594b0696a9932040cdd03c 2024-11-10T18:13:59,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f91104aaf50a4593a1fdfb20da5dc24e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/f91104aaf50a4593a1fdfb20da5dc24e 2024-11-10T18:13:59,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/316c83ff01eb44ccadf7e59c3bba5ec1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/316c83ff01eb44ccadf7e59c3bba5ec1 2024-11-10T18:13:59,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ede3ed027c9a46018cd051574aee2c28 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ede3ed027c9a46018cd051574aee2c28 2024-11-10T18:13:59,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2dd994df19e8441db2b6420b6cc399ee to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2dd994df19e8441db2b6420b6cc399ee 2024-11-10T18:13:59,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6bd6dccff1114a439d114161f84fa479 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6bd6dccff1114a439d114161f84fa479 2024-11-10T18:13:59,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ef397b7ba8bb481a89cf88b10547ea25 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/ef397b7ba8bb481a89cf88b10547ea25 2024-11-10T18:13:59,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/8e47002ffd6449caa4fcd4ebcadd32a2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/8e47002ffd6449caa4fcd4ebcadd32a2 2024-11-10T18:13:59,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b2d263c9bbe2463994521aee7756690f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b2d263c9bbe2463994521aee7756690f 2024-11-10T18:13:59,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3fffdb20212848b684b2de465e3720ca to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3fffdb20212848b684b2de465e3720ca 2024-11-10T18:13:59,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b3423a7ec4ed40749e79b739ca73c737 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b3423a7ec4ed40749e79b739ca73c737 2024-11-10T18:13:59,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/24b420197b84475292bd700d3212a662 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/24b420197b84475292bd700d3212a662 2024-11-10T18:13:59,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1c9d62fa8bfe4fafa25daf3ba510938e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1c9d62fa8bfe4fafa25daf3ba510938e 2024-11-10T18:13:59,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6600ce0070224f0eb8ba2bca3c08ff2e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/6600ce0070224f0eb8ba2bca3c08ff2e 2024-11-10T18:13:59,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2b5dad8f165243ecba19b151f5f4b82f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/2b5dad8f165243ecba19b151f5f4b82f 2024-11-10T18:13:59,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/e8c38cbfdaf04f7eb74ff9a8c0f732bd 2024-11-10T18:13:59,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7e60c20492d4cd08f2dd57c687882a4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7e60c20492d4cd08f2dd57c687882a4 2024-11-10T18:13:59,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b7be4abf476945e28608e1af647e95a8 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/b7be4abf476945e28608e1af647e95a8 2024-11-10T18:13:59,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1b7d44742cd04f1986d403f6a9f2bfb7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/1b7d44742cd04f1986d403f6a9f2bfb7 2024-11-10T18:13:59,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3f1bbe2e3a8646b494a7dc974b0671c4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3f1bbe2e3a8646b494a7dc974b0671c4 2024-11-10T18:13:59,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7260aa3ce6b4454a9f510ab7f486f00 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/d7260aa3ce6b4454a9f510ab7f486f00 2024-11-10T18:13:59,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/20c636898cb6427590467f99ca8fd949 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/20c636898cb6427590467f99ca8fd949 2024-11-10T18:13:59,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/22e9a2fae7de4008bc37f43e0fbe6520 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/22e9a2fae7de4008bc37f43e0fbe6520 2024-11-10T18:13:59,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/07fd4a36d5e44a39ab7aae28872ab027, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/860deb6bd2774f6bbe3e70e8db373855, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/3084c864878848b99d5820f4f5a93b5b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ebe23100885e471bb8a6b1580b158358, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/becd386b51a047ee87b3a780cc1fdf6f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b31ef52b87e643cf8074871bdf708ceb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/102ede0288aa450ba9360fd1d420eafb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a486eefd61e14512be3666cfe5800bd8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/19343b849a7c4f5e8591a7e0ff50d215, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5b26ceca95243a69a28f03c1dcc8699, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/035b2ccaf736426ebd0c27fc7d8424ff, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/568b8c76e32f4843be2ef3f42cac4562, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/e66b52e520da4bfa8e8d46fea83a2dc1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/f7c374ec0a4b4f35bc41e16d1b7297df, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/7add5f14eaca4c62a038c73e62d9944f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c6cca38b002a4148adf0b36bc85a743d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1c782944c8b48318eae66c5c5fe5ded, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fa25846048ef40ecbb960ad8ed3163c2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/dca5da101fc94605bed090f8ea248efd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/25913795810e4f62bd6d6cf738aef400, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/de7b9c233fb64565ab44ca542079de31, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a9c105b0b94841b1a2f2a2bd9fff3d97, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/792128da9f144df88d2fc2f01fd874c6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6ebe4c5b1622445da169e93e88bc34db, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b56a879f3f764cec8529e6d48d245133, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/8de49c135f6149c2a344fd8ece92884b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/494821ae0a9b4ec4b2140e5d8ff4f93f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b221280c050f49638d19567d87d1fd51, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5be076871a64367a595abbdff6ac8b2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ac3ca741a7c8491d8b44628748ef0303, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6919802c0c41411fb1a5d433d72c4dc0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fc36fe32cd7944eb8d6af2bcfe150086, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d242186d02cc41c5b1d6ccd08aafd916, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/32bb538503d3428ca508889af9446319] to archive 2024-11-10T18:13:59,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:13:59,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/07fd4a36d5e44a39ab7aae28872ab027 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/07fd4a36d5e44a39ab7aae28872ab027 2024-11-10T18:13:59,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/860deb6bd2774f6bbe3e70e8db373855 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/860deb6bd2774f6bbe3e70e8db373855 2024-11-10T18:13:59,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/3084c864878848b99d5820f4f5a93b5b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/3084c864878848b99d5820f4f5a93b5b 2024-11-10T18:13:59,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ebe23100885e471bb8a6b1580b158358 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ebe23100885e471bb8a6b1580b158358 2024-11-10T18:13:59,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/becd386b51a047ee87b3a780cc1fdf6f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/becd386b51a047ee87b3a780cc1fdf6f 2024-11-10T18:13:59,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b31ef52b87e643cf8074871bdf708ceb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b31ef52b87e643cf8074871bdf708ceb 2024-11-10T18:13:59,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/102ede0288aa450ba9360fd1d420eafb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/102ede0288aa450ba9360fd1d420eafb 2024-11-10T18:13:59,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a486eefd61e14512be3666cfe5800bd8 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a486eefd61e14512be3666cfe5800bd8 2024-11-10T18:13:59,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/19343b849a7c4f5e8591a7e0ff50d215 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/19343b849a7c4f5e8591a7e0ff50d215 2024-11-10T18:13:59,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5b26ceca95243a69a28f03c1dcc8699 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5b26ceca95243a69a28f03c1dcc8699 2024-11-10T18:13:59,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/035b2ccaf736426ebd0c27fc7d8424ff to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/035b2ccaf736426ebd0c27fc7d8424ff 2024-11-10T18:13:59,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/568b8c76e32f4843be2ef3f42cac4562 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/568b8c76e32f4843be2ef3f42cac4562 2024-11-10T18:13:59,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/e66b52e520da4bfa8e8d46fea83a2dc1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/e66b52e520da4bfa8e8d46fea83a2dc1 2024-11-10T18:13:59,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/f7c374ec0a4b4f35bc41e16d1b7297df to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/f7c374ec0a4b4f35bc41e16d1b7297df 2024-11-10T18:13:59,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/7add5f14eaca4c62a038c73e62d9944f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/7add5f14eaca4c62a038c73e62d9944f 2024-11-10T18:13:59,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c6cca38b002a4148adf0b36bc85a743d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c6cca38b002a4148adf0b36bc85a743d 2024-11-10T18:13:59,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1c782944c8b48318eae66c5c5fe5ded to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1c782944c8b48318eae66c5c5fe5ded 2024-11-10T18:13:59,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fa25846048ef40ecbb960ad8ed3163c2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fa25846048ef40ecbb960ad8ed3163c2 2024-11-10T18:13:59,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/dca5da101fc94605bed090f8ea248efd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/dca5da101fc94605bed090f8ea248efd 2024-11-10T18:13:59,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/25913795810e4f62bd6d6cf738aef400 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/25913795810e4f62bd6d6cf738aef400 2024-11-10T18:13:59,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/de7b9c233fb64565ab44ca542079de31 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/de7b9c233fb64565ab44ca542079de31 2024-11-10T18:13:59,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a9c105b0b94841b1a2f2a2bd9fff3d97 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/a9c105b0b94841b1a2f2a2bd9fff3d97 2024-11-10T18:13:59,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/792128da9f144df88d2fc2f01fd874c6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/792128da9f144df88d2fc2f01fd874c6 2024-11-10T18:13:59,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6ebe4c5b1622445da169e93e88bc34db to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6ebe4c5b1622445da169e93e88bc34db 2024-11-10T18:13:59,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b56a879f3f764cec8529e6d48d245133 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b56a879f3f764cec8529e6d48d245133 2024-11-10T18:13:59,269 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/8de49c135f6149c2a344fd8ece92884b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/8de49c135f6149c2a344fd8ece92884b 2024-11-10T18:13:59,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/494821ae0a9b4ec4b2140e5d8ff4f93f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/494821ae0a9b4ec4b2140e5d8ff4f93f 2024-11-10T18:13:59,271 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b221280c050f49638d19567d87d1fd51 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/b221280c050f49638d19567d87d1fd51 2024-11-10T18:13:59,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5be076871a64367a595abbdff6ac8b2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/c5be076871a64367a595abbdff6ac8b2 2024-11-10T18:13:59,274 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ac3ca741a7c8491d8b44628748ef0303 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/ac3ca741a7c8491d8b44628748ef0303 2024-11-10T18:13:59,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6919802c0c41411fb1a5d433d72c4dc0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/6919802c0c41411fb1a5d433d72c4dc0 2024-11-10T18:13:59,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fc36fe32cd7944eb8d6af2bcfe150086 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fc36fe32cd7944eb8d6af2bcfe150086 2024-11-10T18:13:59,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d242186d02cc41c5b1d6ccd08aafd916 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d242186d02cc41c5b1d6ccd08aafd916 2024-11-10T18:13:59,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/32bb538503d3428ca508889af9446319 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/32bb538503d3428ca508889af9446319 2024-11-10T18:13:59,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/5d39316a66ad4403928a0b913dcc61d1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/61209753cc034d3da68b30a8512bd6bd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8992cfd18283428f9e17310b4977fe4c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d196d4a096694c52b567cb5d67350821, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/f80dfecb2d054ca0abf128024efd62c1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82161791e53242aca78fc678ca7bcccb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/79aa5fec2b5d4600852b7091d674f519, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a23e46a5d4f847b399578f84886ff4e4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/15a35ad4b4f54734b346a37586756e58, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/ead91e33fcb44041884b5d229ec7d0f3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/38d825f018574ce7aa896ce74273498c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/89763a2cc2a0496eaab3c2de1e967738, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8e29f921f9d148dfa7f2726b3a052cc1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a7a9fe73a4a841a789c25ba47cc654cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b10b0ed877d14658b8e00c5be0a76fe9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1d83b75f5e5540809289c1559fb4d488, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/895fe839fd0241529302d91c02b3c726, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2bfd4c632e864529aadeb51c471a09af, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/9e39509a8b6a4c519fc1681acdc3be01, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/cea87f9c0d60400ab2f1370278b51654, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/673f089165174633bb6ef9567620b349, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d1ff8ac0eb7f4f40b06c698cfcc07150, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b74d27ad6f4f4654ad33e7e194c82130, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a8825bdaaf9e4a59ae7d8820394ef136, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7a793365a66547d28806270d0e1c6245, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/0dc1ece1f61e4a6c8ac8e5530bab34c2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2160b2d9a8874713866448ac0f81e2b1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/40294d96b0d44444b145039e4ef824b4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1637344a924d4b67867f64cf40674cb7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/c7d910dc5a6142329750e05030af730a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/62fc4b05e1084b989ec276c64c193617, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/eada90e34f534ff1909fad051ae10572, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e341edb8e36a488c8fa9b6ef8b0066c4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e4208f0add0741c29cbd60eaa35df22c] to archive 2024-11-10T18:13:59,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:13:59,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/5d39316a66ad4403928a0b913dcc61d1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/5d39316a66ad4403928a0b913dcc61d1 2024-11-10T18:13:59,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/61209753cc034d3da68b30a8512bd6bd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/61209753cc034d3da68b30a8512bd6bd 2024-11-10T18:13:59,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8992cfd18283428f9e17310b4977fe4c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8992cfd18283428f9e17310b4977fe4c 2024-11-10T18:13:59,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d196d4a096694c52b567cb5d67350821 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d196d4a096694c52b567cb5d67350821 2024-11-10T18:13:59,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/f80dfecb2d054ca0abf128024efd62c1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/f80dfecb2d054ca0abf128024efd62c1 2024-11-10T18:13:59,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82161791e53242aca78fc678ca7bcccb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82161791e53242aca78fc678ca7bcccb 2024-11-10T18:13:59,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/79aa5fec2b5d4600852b7091d674f519 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/79aa5fec2b5d4600852b7091d674f519 2024-11-10T18:13:59,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a23e46a5d4f847b399578f84886ff4e4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a23e46a5d4f847b399578f84886ff4e4 2024-11-10T18:13:59,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/15a35ad4b4f54734b346a37586756e58 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/15a35ad4b4f54734b346a37586756e58 2024-11-10T18:13:59,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/ead91e33fcb44041884b5d229ec7d0f3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/ead91e33fcb44041884b5d229ec7d0f3 2024-11-10T18:13:59,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/38d825f018574ce7aa896ce74273498c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/38d825f018574ce7aa896ce74273498c 2024-11-10T18:13:59,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/89763a2cc2a0496eaab3c2de1e967738 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/89763a2cc2a0496eaab3c2de1e967738 2024-11-10T18:13:59,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8e29f921f9d148dfa7f2726b3a052cc1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/8e29f921f9d148dfa7f2726b3a052cc1 2024-11-10T18:13:59,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a7a9fe73a4a841a789c25ba47cc654cd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a7a9fe73a4a841a789c25ba47cc654cd 2024-11-10T18:13:59,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b10b0ed877d14658b8e00c5be0a76fe9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b10b0ed877d14658b8e00c5be0a76fe9 2024-11-10T18:13:59,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1d83b75f5e5540809289c1559fb4d488 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1d83b75f5e5540809289c1559fb4d488 2024-11-10T18:13:59,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/895fe839fd0241529302d91c02b3c726 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/895fe839fd0241529302d91c02b3c726 2024-11-10T18:13:59,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2bfd4c632e864529aadeb51c471a09af to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2bfd4c632e864529aadeb51c471a09af 2024-11-10T18:13:59,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/9e39509a8b6a4c519fc1681acdc3be01 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/9e39509a8b6a4c519fc1681acdc3be01 2024-11-10T18:13:59,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/cea87f9c0d60400ab2f1370278b51654 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/cea87f9c0d60400ab2f1370278b51654 2024-11-10T18:13:59,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/673f089165174633bb6ef9567620b349 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/673f089165174633bb6ef9567620b349 2024-11-10T18:13:59,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d1ff8ac0eb7f4f40b06c698cfcc07150 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/d1ff8ac0eb7f4f40b06c698cfcc07150 2024-11-10T18:13:59,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b74d27ad6f4f4654ad33e7e194c82130 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/b74d27ad6f4f4654ad33e7e194c82130 2024-11-10T18:13:59,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a8825bdaaf9e4a59ae7d8820394ef136 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/a8825bdaaf9e4a59ae7d8820394ef136 2024-11-10T18:13:59,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7a793365a66547d28806270d0e1c6245 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7a793365a66547d28806270d0e1c6245 2024-11-10T18:13:59,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/0dc1ece1f61e4a6c8ac8e5530bab34c2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/0dc1ece1f61e4a6c8ac8e5530bab34c2 2024-11-10T18:13:59,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2160b2d9a8874713866448ac0f81e2b1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/2160b2d9a8874713866448ac0f81e2b1 2024-11-10T18:13:59,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/40294d96b0d44444b145039e4ef824b4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/40294d96b0d44444b145039e4ef824b4 2024-11-10T18:13:59,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1637344a924d4b67867f64cf40674cb7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/1637344a924d4b67867f64cf40674cb7 2024-11-10T18:13:59,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/c7d910dc5a6142329750e05030af730a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/c7d910dc5a6142329750e05030af730a 2024-11-10T18:13:59,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/62fc4b05e1084b989ec276c64c193617 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/62fc4b05e1084b989ec276c64c193617 2024-11-10T18:13:59,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/eada90e34f534ff1909fad051ae10572 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/eada90e34f534ff1909fad051ae10572 2024-11-10T18:13:59,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e341edb8e36a488c8fa9b6ef8b0066c4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e341edb8e36a488c8fa9b6ef8b0066c4 2024-11-10T18:13:59,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e4208f0add0741c29cbd60eaa35df22c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/e4208f0add0741c29cbd60eaa35df22c 2024-11-10T18:13:59,331 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/recovered.edits/633.seqid, newMaxSeqId=633, maxSeqId=1 2024-11-10T18:13:59,332 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. 2024-11-10T18:13:59,332 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 17351107c8cbd49315660faf09692c94: Waiting for close lock at 1731262437909Running coprocessor pre-close hooks at 1731262437909Disabling compacts and flushes for region at 1731262437909Disabling writes for close at 1731262437910 (+1 ms)Obtaining lock to block concurrent updates at 1731262437910Preparing flush snapshotting stores in 17351107c8cbd49315660faf09692c94 at 1731262437910Finished memstore snapshotting TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731262437910Flushing stores of TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94. at 1731262437911 (+1 ms)Flushing 17351107c8cbd49315660faf09692c94/A: creating writer at 1731262437911Flushing 17351107c8cbd49315660faf09692c94/A: appending metadata at 1731262437914 (+3 ms)Flushing 17351107c8cbd49315660faf09692c94/A: closing flushed file at 1731262437914Flushing 17351107c8cbd49315660faf09692c94/B: creating writer at 1731262438324 (+410 ms)Flushing 17351107c8cbd49315660faf09692c94/B: appending metadata at 1731262438327 (+3 ms)Flushing 17351107c8cbd49315660faf09692c94/B: closing flushed file at 1731262438327Flushing 17351107c8cbd49315660faf09692c94/C: creating writer at 1731262438736 (+409 ms)Flushing 17351107c8cbd49315660faf09692c94/C: appending metadata at 1731262438739 (+3 ms)Flushing 17351107c8cbd49315660faf09692c94/C: closing flushed file at 1731262438739Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f532b1e: reopening flushed file at 1731262439149 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57d5eae2: reopening flushed file at 1731262439155 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b6ecb6c: reopening flushed file at 1731262439161 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 17351107c8cbd49315660faf09692c94 in 1257ms, sequenceid=630, compaction requested=true at 1731262439167 (+6 ms)Writing region close event to WAL at 1731262439327 (+160 ms)Running coprocessor post-close hooks at 1731262439332 (+5 ms)Closed at 1731262439332 2024-11-10T18:13:59,335 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 17351107c8cbd49315660faf09692c94 2024-11-10T18:13:59,336 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=17351107c8cbd49315660faf09692c94, regionState=CLOSED 2024-11-10T18:13:59,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:13:59,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-10T18:13:59,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 17351107c8cbd49315660faf09692c94, server=691ea56905d2,32909,1731262402364 in 1.5870 sec 2024-11-10T18:13:59,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-10T18:13:59,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=17351107c8cbd49315660faf09692c94, UNASSIGN in 1.5930 sec 2024-11-10T18:13:59,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-10T18:13:59,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6000 sec 2024-11-10T18:13:59,347 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262439347"}]},"ts":"1731262439347"} 2024-11-10T18:13:59,349 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T18:13:59,349 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T18:13:59,351 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6150 sec 2024-11-10T18:13:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-10T18:13:59,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:13:59,889 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T18:13:59,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:59,894 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:59,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T18:13:59,896 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:59,900 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94 2024-11-10T18:13:59,904 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/recovered.edits] 2024-11-10T18:13:59,907 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3efd40ac4e9b42d89365861879d859dc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/3efd40ac4e9b42d89365861879d859dc 2024-11-10T18:13:59,908 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/c02463156698482998dda59397301dd3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/c02463156698482998dda59397301dd3 2024-11-10T18:13:59,910 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/cb65e38ef5b141ac90b59c1860b9c605 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/A/cb65e38ef5b141ac90b59c1860b9c605 2024-11-10T18:13:59,912 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1f4eb176754458296bd2d414bd2d7da to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d1f4eb176754458296bd2d414bd2d7da 2024-11-10T18:13:59,913 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d6415e491ae24fef9fc5ddfb4c5003c1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/d6415e491ae24fef9fc5ddfb4c5003c1 2024-11-10T18:13:59,914 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fd2936665d26449e8171f8f83744c150 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/B/fd2936665d26449e8171f8f83744c150 2024-11-10T18:13:59,917 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/4895e96195d64b9dadcd8ff1d9e100ae to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/4895e96195d64b9dadcd8ff1d9e100ae 2024-11-10T18:13:59,918 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7361cdbeb7f840c4a0299c1e9e0fdb28 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/7361cdbeb7f840c4a0299c1e9e0fdb28 2024-11-10T18:13:59,919 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82153fb9bfe743a6a0524a7540b20c23 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/C/82153fb9bfe743a6a0524a7540b20c23 2024-11-10T18:13:59,923 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/recovered.edits/633.seqid to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94/recovered.edits/633.seqid 2024-11-10T18:13:59,923 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/17351107c8cbd49315660faf09692c94 2024-11-10T18:13:59,923 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T18:13:59,929 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:59,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-10T18:13:59,939 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T18:13:59,974 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T18:13:59,976 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:59,976 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T18:13:59,976 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731262439976"}]},"ts":"9223372036854775807"} 2024-11-10T18:13:59,981 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T18:13:59,981 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 17351107c8cbd49315660faf09692c94, NAME => 'TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T18:13:59,981 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T18:13:59,981 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731262439981"}]},"ts":"9223372036854775807"} 2024-11-10T18:13:59,984 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T18:13:59,985 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:13:59,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 96 msec 2024-11-10T18:14:00,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-10T18:14:00,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T18:14:00,006 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:00,017 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/691ea56905d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;691ea56905d2:32909-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1086771773_22 at /127.0.0.1:59838 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=447 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=343 (was 267) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8208 (was 8747) 2024-11-10T18:14:00,026 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=343, ProcessCount=11, AvailableMemoryMB=8207 2024-11-10T18:14:00,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:14:00,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:14:00,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:00,030 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T18:14:00,030 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:00,030 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-10T18:14:00,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T18:14:00,031 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T18:14:00,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741948_1124 (size=1061) 2024-11-10T18:14:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T18:14:00,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T18:14:00,440 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:14:00,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741949_1125 (size=53) 2024-11-10T18:14:00,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T18:14:00,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:14:00,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a31b277868914d792297968bfe99e4fc, disabling compactions & flushes 2024-11-10T18:14:00,847 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:00,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:00,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. after waiting 0 ms 2024-11-10T18:14:00,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:00,847 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:00,847 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a31b277868914d792297968bfe99e4fc: Waiting for close lock at 1731262440847Disabling compacts and flushes for region at 1731262440847Disabling writes for close at 1731262440847Writing region close event to WAL at 1731262440847Closed at 1731262440847 2024-11-10T18:14:00,848 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T18:14:00,849 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731262440848"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731262440848"}]},"ts":"1731262440848"} 2024-11-10T18:14:00,851 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T18:14:00,852 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T18:14:00,852 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262440852"}]},"ts":"1731262440852"} 2024-11-10T18:14:00,854 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T18:14:00,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, ASSIGN}] 2024-11-10T18:14:00,855 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, ASSIGN 2024-11-10T18:14:00,856 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, ASSIGN; state=OFFLINE, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=false 2024-11-10T18:14:01,006 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:01,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, ASSIGN because future has completed 2024-11-10T18:14:01,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:14:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T18:14:01,165 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:01,165 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:14:01,165 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,165 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:14:01,165 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,165 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,167 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,168 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:01,169 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a31b277868914d792297968bfe99e4fc columnFamilyName A 2024-11-10T18:14:01,169 DEBUG [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:01,169 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(327): Store=a31b277868914d792297968bfe99e4fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:01,170 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,171 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:01,171 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a31b277868914d792297968bfe99e4fc columnFamilyName B 2024-11-10T18:14:01,171 DEBUG [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:01,171 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(327): Store=a31b277868914d792297968bfe99e4fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:01,171 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,172 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:01,172 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a31b277868914d792297968bfe99e4fc columnFamilyName C 2024-11-10T18:14:01,173 DEBUG [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:01,173 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(327): Store=a31b277868914d792297968bfe99e4fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:01,173 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:01,173 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,174 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,174 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,175 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,175 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,175 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:14:01,176 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,178 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:14:01,179 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened a31b277868914d792297968bfe99e4fc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63654346, jitterRate=-0.05147632956504822}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:14:01,179 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:01,180 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for a31b277868914d792297968bfe99e4fc: Running coprocessor pre-open hook at 1731262441166Writing region info on filesystem at 1731262441166Initializing all the Stores at 1731262441166Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262441166Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262441167 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262441167Cleaning up temporary data from old regions at 1731262441175 (+8 ms)Running coprocessor post-open hooks at 1731262441179 (+4 ms)Region opened successfully at 1731262441180 (+1 ms) 2024-11-10T18:14:01,180 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., pid=32, masterSystemTime=1731262441161 2024-11-10T18:14:01,182 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:01,182 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:01,183 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:01,184 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:14:01,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-10T18:14:01,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 in 176 msec 2024-11-10T18:14:01,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-10T18:14:01,189 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, ASSIGN in 333 msec 2024-11-10T18:14:01,190 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T18:14:01,190 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262441190"}]},"ts":"1731262441190"} 2024-11-10T18:14:01,191 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T18:14:01,192 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T18:14:01,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1640 sec 2024-11-10T18:14:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-10T18:14:02,175 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:02,176 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:02,176 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:02,176 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:02,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:02,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c970665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:02,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:02,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:02,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655d0d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:02,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:02,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:02,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:02,186 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:14:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:14:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ed75f2b 2024-11-10T18:14:02,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:14:02,187 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:14:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:14:02,193 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:14:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:02,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741950_1126 (size=1097) 2024-11-10T18:14:02,612 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T18:14:02,613 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T18:14:02,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:14:02,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, REOPEN/MOVE}] 2024-11-10T18:14:02,624 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, REOPEN/MOVE 2024-11-10T18:14:02,624 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:02,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, REOPEN/MOVE because future has completed 2024-11-10T18:14:02,627 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:14:02,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:14:02,779 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:02,779 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:14:02,779 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing a31b277868914d792297968bfe99e4fc, disabling compactions & flushes 2024-11-10T18:14:02,779 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:02,779 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:02,779 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. after waiting 0 ms 2024-11-10T18:14:02,780 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:02,784 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T18:14:02,784 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:02,784 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for a31b277868914d792297968bfe99e4fc: Waiting for close lock at 1731262442779Running coprocessor pre-close hooks at 1731262442779Disabling compacts and flushes for region at 1731262442779Disabling writes for close at 1731262442779Writing region close event to WAL at 1731262442780 (+1 ms)Running coprocessor post-close hooks at 1731262442784 (+4 ms)Closed at 1731262442784 2024-11-10T18:14:02,785 WARN [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: a31b277868914d792297968bfe99e4fc to self. 2024-11-10T18:14:02,787 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:02,787 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=CLOSED 2024-11-10T18:14:02,789 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:14:02,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-10T18:14:02,791 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 in 163 msec 2024-11-10T18:14:02,792 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, REOPEN/MOVE; state=CLOSED, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=true 2024-11-10T18:14:02,942 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:02,944 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, REOPEN/MOVE because future has completed 2024-11-10T18:14:02,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:14:03,100 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,100 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:14:03,100 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,100 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:14:03,100 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,100 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,103 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,103 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:03,109 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a31b277868914d792297968bfe99e4fc columnFamilyName A 2024-11-10T18:14:03,111 DEBUG [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:03,111 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(327): Store=a31b277868914d792297968bfe99e4fc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:03,112 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,112 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:03,113 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a31b277868914d792297968bfe99e4fc columnFamilyName B 2024-11-10T18:14:03,113 DEBUG [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:03,114 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(327): Store=a31b277868914d792297968bfe99e4fc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:03,114 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,114 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:03,115 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a31b277868914d792297968bfe99e4fc columnFamilyName C 2024-11-10T18:14:03,115 DEBUG [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:03,115 INFO [StoreOpener-a31b277868914d792297968bfe99e4fc-1 {}] regionserver.HStore(327): Store=a31b277868914d792297968bfe99e4fc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:03,115 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,115 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,116 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,117 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,118 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,118 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,118 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:14:03,119 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,120 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened a31b277868914d792297968bfe99e4fc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69719919, jitterRate=0.03890775144100189}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:14:03,120 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,122 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for a31b277868914d792297968bfe99e4fc: Running coprocessor pre-open hook at 1731262443101Writing region info on filesystem at 1731262443101Initializing all the Stores at 1731262443101Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262443101Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262443101Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262443101Cleaning up temporary data from old regions at 1731262443118 (+17 ms)Running coprocessor post-open hooks at 1731262443120 (+2 ms)Region opened successfully at 1731262443122 (+2 ms) 2024-11-10T18:14:03,123 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., pid=37, masterSystemTime=1731262443096 2024-11-10T18:14:03,125 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,125 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,125 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=OPEN, openSeqNum=5, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:14:03,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-10T18:14:03,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 in 184 msec 2024-11-10T18:14:03,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-10T18:14:03,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, REOPEN/MOVE in 506 msec 2024-11-10T18:14:03,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-10T18:14:03,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-10T18:14:03,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 940 msec 2024-11-10T18:14:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-10T18:14:03,139 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:03,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,140 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679b330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,141 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,142 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,143 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd7ed2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,144 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,144 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,144 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,145 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,145 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,146 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37846, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac3782b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,148 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@347ae191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,149 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,150 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,150 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c31c122, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,152 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c54ff46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4315a40e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,155 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,155 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e0cbee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,156 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6b146e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5ced64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,160 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37900, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23c9013f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,161 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219f80f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,163 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ac8dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,164 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,164 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6f899f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,166 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76a3785b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:03,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:03,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:03,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@701f7ad6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:03,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:03,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:03,169 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30213712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:03,170 DEBUG [Thread-640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,170 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,170 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,170 DEBUG [Thread-653 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,170 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:03,171 DEBUG [Thread-649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,171 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,172 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,173 DEBUG [Thread-647 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,173 DEBUG [Thread-647 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,173 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-10T18:14:03,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,174 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42884, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T18:14:03,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,175 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:03,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,183 DEBUG [Thread-651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,186 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,186 DEBUG [Thread-642 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:03,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:03,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:03,190 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:03,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:14:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:03,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:03,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731262443207, max timestamp=1731262443228Num uniques 859; 2024-11-10T18:14:03,236 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T18:14:03,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731262443207, max timestamp=1731262443228Num uniques 859; 2024-11-10T18:14:03,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T18:14:03,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731262443207, max timestamp=1731262443228Num uniques 859; 2024-11-10T18:14:03,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-10T18:14:03,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be74515d7b0749cfa8faf38a2e6d3fe2_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262443190/Put/seqid=0 2024-11-10T18:14:03,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741951_1127 (size=12154) 2024-11-10T18:14:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T18:14:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262503269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262503271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,326 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,333 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262503325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,340 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262503325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262503325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262503435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262503435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262503445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262503445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262503445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T18:14:03,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,643 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262503645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262503645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,671 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:03,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,678 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be74515d7b0749cfa8faf38a2e6d3fe2_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be74515d7b0749cfa8faf38a2e6d3fe2_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:03,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262503676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262503676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262503676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,681 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/c585a1111e6d4d4f898ef9f8b340f052, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:03,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/c585a1111e6d4d4f898ef9f8b340f052 is 175, key is test_row_0/A:col10/1731262443190/Put/seqid=0 2024-11-10T18:14:03,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741952_1128 (size=30955) 2024-11-10T18:14:03,720 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/c585a1111e6d4d4f898ef9f8b340f052 2024-11-10T18:14:03,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/0bbed838443a4f07ac48dfb76490b8a2 is 50, key is test_row_0/B:col10/1731262443190/Put/seqid=0 2024-11-10T18:14:03,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741953_1129 (size=12001) 2024-11-10T18:14:03,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/0bbed838443a4f07ac48dfb76490b8a2 2024-11-10T18:14:03,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T18:14:03,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/abb33989671a44d1aa36353021f0a3d5 is 50, key is test_row_0/C:col10/1731262443190/Put/seqid=0 2024-11-10T18:14:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741954_1130 (size=12001) 2024-11-10T18:14:03,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:03,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:03,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262503955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262503956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,959 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262503984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262503985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:03,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262503985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:03,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:03,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:04,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:04,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:04,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:04,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:04,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/abb33989671a44d1aa36353021f0a3d5 2024-11-10T18:14:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/c585a1111e6d4d4f898ef9f8b340f052 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052 2024-11-10T18:14:04,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052, entries=150, sequenceid=15, filesize=30.2 K 2024-11-10T18:14:04,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/0bbed838443a4f07ac48dfb76490b8a2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/0bbed838443a4f07ac48dfb76490b8a2 2024-11-10T18:14:04,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/0bbed838443a4f07ac48dfb76490b8a2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T18:14:04,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/abb33989671a44d1aa36353021f0a3d5 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/abb33989671a44d1aa36353021f0a3d5 2024-11-10T18:14:04,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:04,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:04,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:04,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:04,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:04,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:04,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/abb33989671a44d1aa36353021f0a3d5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-10T18:14:04,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=147.60 KB/151140 for a31b277868914d792297968bfe99e4fc in 1080ms, sequenceid=15, compaction requested=false 2024-11-10T18:14:04,270 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T18:14:04,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:04,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T18:14:04,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-10T18:14:04,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:04,416 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-11-10T18:14:04,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:04,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:04,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:04,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:04,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:04,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:04,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be40c3a317e740498712b522333f279e_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262443291/Put/seqid=0 2024-11-10T18:14:04,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741955_1131 (size=12154) 2024-11-10T18:14:04,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:04,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:04,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:04,497 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110be40c3a317e740498712b522333f279e_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be40c3a317e740498712b522333f279e_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:04,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262504495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/21e24fc511774f7faffca8800d631b3c, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:04,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262504495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/21e24fc511774f7faffca8800d631b3c is 175, key is test_row_0/A:col10/1731262443291/Put/seqid=0 2024-11-10T18:14:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262504496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262504501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262504504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741956_1132 (size=30955) 2024-11-10T18:14:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,525 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/21e24fc511774f7faffca8800d631b3c 2024-11-10T18:14:04,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/95adc40d2cc34084af3b2481dbc22bb1 is 50, key is test_row_0/B:col10/1731262443291/Put/seqid=0 2024-11-10T18:14:04,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741957_1133 (size=12001) 2024-11-10T18:14:04,556 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/95adc40d2cc34084af3b2481dbc22bb1 2024-11-10T18:14:04,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/e6c023cc261744b281b2457f2af1c216 is 50, key is test_row_0/C:col10/1731262443291/Put/seqid=0 2024-11-10T18:14:04,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741958_1134 (size=12001) 2024-11-10T18:14:04,591 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/e6c023cc261744b281b2457f2af1c216 2024-11-10T18:14:04,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/21e24fc511774f7faffca8800d631b3c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c 2024-11-10T18:14:04,606 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c, entries=150, sequenceid=49, filesize=30.2 K 2024-11-10T18:14:04,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/95adc40d2cc34084af3b2481dbc22bb1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/95adc40d2cc34084af3b2481dbc22bb1 2024-11-10T18:14:04,617 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/95adc40d2cc34084af3b2481dbc22bb1, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T18:14:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262504614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262504615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/e6c023cc261744b281b2457f2af1c216 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/e6c023cc261744b281b2457f2af1c216 2024-11-10T18:14:04,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262504625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,631 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,638 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/e6c023cc261744b281b2457f2af1c216, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T18:14:04,640 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=46.96 KB/48090 for a31b277868914d792297968bfe99e4fc in 225ms, sequenceid=49, compaction requested=false 2024-11-10T18:14:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:04,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-10T18:14:04,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-10T18:14:04,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-10T18:14:04,645 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4630 sec 2024-11-10T18:14:04,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.4780 sec 2024-11-10T18:14:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:04,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:14:04,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:04,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:04,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:04,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:04,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:04,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:04,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111082fa4446e558432cbb421996ff5d45f8_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:04,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741959_1135 (size=14594) 2024-11-10T18:14:04,874 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:14:04,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:04,882 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111082fa4446e558432cbb421996ff5d45f8_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111082fa4446e558432cbb421996ff5d45f8_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:04,883 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d556255a5c924e0f96b54fc350685c98, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:04,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d556255a5c924e0f96b54fc350685c98 is 175, key is test_row_0/A:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:04,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741960_1136 (size=39549) 2024-11-10T18:14:04,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262504910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262504919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:04,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262504921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262505035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262505035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262505035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262505245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262505245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,252 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262505245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,297 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d556255a5c924e0f96b54fc350685c98 2024-11-10T18:14:05,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/94355c6695cc4ba9a5c7ed9217542dbb is 50, key is test_row_0/B:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:05,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741961_1137 (size=12001) 2024-11-10T18:14:05,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/94355c6695cc4ba9a5c7ed9217542dbb 2024-11-10T18:14:05,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/c0c9fa812f1643d6a7bbb863bc64a3cc is 50, key is test_row_0/C:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:05,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741962_1138 (size=12001) 2024-11-10T18:14:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-10T18:14:05,335 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:05,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:05,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/c0c9fa812f1643d6a7bbb863bc64a3cc 2024-11-10T18:14:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-10T18:14:05,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T18:14:05,338 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:05,339 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:05,339 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:05,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d556255a5c924e0f96b54fc350685c98 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98 2024-11-10T18:14:05,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98, entries=200, sequenceid=61, filesize=38.6 K 2024-11-10T18:14:05,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/94355c6695cc4ba9a5c7ed9217542dbb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/94355c6695cc4ba9a5c7ed9217542dbb 2024-11-10T18:14:05,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/94355c6695cc4ba9a5c7ed9217542dbb, entries=150, sequenceid=61, filesize=11.7 K 2024-11-10T18:14:05,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/c0c9fa812f1643d6a7bbb863bc64a3cc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c0c9fa812f1643d6a7bbb863bc64a3cc 2024-11-10T18:14:05,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c0c9fa812f1643d6a7bbb863bc64a3cc, entries=150, sequenceid=61, filesize=11.7 K 2024-11-10T18:14:05,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a31b277868914d792297968bfe99e4fc in 547ms, sequenceid=61, compaction requested=true 2024-11-10T18:14:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:05,375 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:05,375 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:05,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:05,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:05,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:05,376 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:05,377 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:05,377 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:05,377 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:05,377 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=99.1 K 2024-11-10T18:14:05,377 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:05,377 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98] 2024-11-10T18:14:05,378 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:05,378 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:05,378 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:05,378 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/0bbed838443a4f07ac48dfb76490b8a2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/95adc40d2cc34084af3b2481dbc22bb1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/94355c6695cc4ba9a5c7ed9217542dbb] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=35.2 K 2024-11-10T18:14:05,378 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting c585a1111e6d4d4f898ef9f8b340f052, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731262443184 2024-11-10T18:14:05,379 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bbed838443a4f07ac48dfb76490b8a2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731262443184 2024-11-10T18:14:05,379 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21e24fc511774f7faffca8800d631b3c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262443263 2024-11-10T18:14:05,379 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 95adc40d2cc34084af3b2481dbc22bb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262443263 2024-11-10T18:14:05,380 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d556255a5c924e0f96b54fc350685c98, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731262444481 2024-11-10T18:14:05,380 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 94355c6695cc4ba9a5c7ed9217542dbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731262444499 2024-11-10T18:14:05,391 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#120 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:05,392 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d38200c0a6f34347855094d252c7271e is 50, key is test_row_0/B:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:05,401 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:05,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741963_1139 (size=12104) 2024-11-10T18:14:05,412 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100de0f870075943ce8abb8d6a10f080bf_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:05,419 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100de0f870075943ce8abb8d6a10f080bf_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:05,419 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100de0f870075943ce8abb8d6a10f080bf_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:05,427 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d38200c0a6f34347855094d252c7271e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d38200c0a6f34347855094d252c7271e 2024-11-10T18:14:05,436 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into d38200c0a6f34347855094d252c7271e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:05,436 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:05,436 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=13, startTime=1731262445375; duration=0sec 2024-11-10T18:14:05,437 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:05,437 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:05,437 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:05,439 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:05,439 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:05,439 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:05,439 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/abb33989671a44d1aa36353021f0a3d5, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/e6c023cc261744b281b2457f2af1c216, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c0c9fa812f1643d6a7bbb863bc64a3cc] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=35.2 K 2024-11-10T18:14:05,440 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting abb33989671a44d1aa36353021f0a3d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731262443184 2024-11-10T18:14:05,441 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c023cc261744b281b2457f2af1c216, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262443263 2024-11-10T18:14:05,442 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c0c9fa812f1643d6a7bbb863bc64a3cc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731262444499 2024-11-10T18:14:05,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T18:14:05,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741964_1140 (size=4469) 2024-11-10T18:14:05,478 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:05,479 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/cf0c63152cf249b58d1b116503a37a1d is 50, key is test_row_0/C:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:05,481 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#121 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:05,482 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/aa8abbac578a46958f3c1f42f884acb0 is 175, key is test_row_0/A:col10/1731262444499/Put/seqid=0 2024-11-10T18:14:05,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-10T18:14:05,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:05,494 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:14:05,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741965_1141 (size=12104) 2024-11-10T18:14:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:05,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:05,501 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/cf0c63152cf249b58d1b116503a37a1d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/cf0c63152cf249b58d1b116503a37a1d 2024-11-10T18:14:05,514 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into cf0c63152cf249b58d1b116503a37a1d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:05,515 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:05,515 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=13, startTime=1731262445376; duration=0sec 2024-11-10T18:14:05,515 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:05,515 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:05,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:05,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:05,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741966_1142 (size=31058) 2024-11-10T18:14:05,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110900f64e764b647558195355b7ea4fc8a_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262444879/Put/seqid=0 2024-11-10T18:14:05,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262505547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262505556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741967_1143 (size=12154) 2024-11-10T18:14:05,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:05,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262505565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262505565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,569 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110900f64e764b647558195355b7ea4fc8a_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110900f64e764b647558195355b7ea4fc8a_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:05,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262505565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:05,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e is 175, key is test_row_0/A:col10/1731262444879/Put/seqid=0 2024-11-10T18:14:05,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741968_1144 (size=30955) 2024-11-10T18:14:05,585 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=82, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e 2024-11-10T18:14:05,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/93126532b8c544eaab6f7281268d8428 is 50, key is test_row_0/B:col10/1731262444879/Put/seqid=0 2024-11-10T18:14:05,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741969_1145 (size=12001) 2024-11-10T18:14:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T18:14:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262505665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262505665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262505875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262505875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:05,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:05,877 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:05,932 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/aa8abbac578a46958f3c1f42f884acb0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/aa8abbac578a46958f3c1f42f884acb0 2024-11-10T18:14:05,940 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into aa8abbac578a46958f3c1f42f884acb0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:05,940 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:05,940 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=13, startTime=1731262445375; duration=0sec 2024-11-10T18:14:05,940 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:05,940 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:05,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T18:14:06,012 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/93126532b8c544eaab6f7281268d8428 2024-11-10T18:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/3d86faa4983842b589ae3103834e2b8e is 50, key is test_row_0/C:col10/1731262444879/Put/seqid=0 2024-11-10T18:14:06,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741970_1146 (size=12001) 2024-11-10T18:14:06,052 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/3d86faa4983842b589ae3103834e2b8e 2024-11-10T18:14:06,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e 2024-11-10T18:14:06,073 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e, entries=150, sequenceid=82, filesize=30.2 K 2024-11-10T18:14:06,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/93126532b8c544eaab6f7281268d8428 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/93126532b8c544eaab6f7281268d8428 2024-11-10T18:14:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262506075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,080 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/93126532b8c544eaab6f7281268d8428, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T18:14:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262506075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262506075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,082 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/3d86faa4983842b589ae3103834e2b8e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d86faa4983842b589ae3103834e2b8e 2024-11-10T18:14:06,090 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d86faa4983842b589ae3103834e2b8e, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T18:14:06,091 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a31b277868914d792297968bfe99e4fc in 598ms, sequenceid=82, compaction requested=false 2024-11-10T18:14:06,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:06,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-10T18:14:06,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-10T18:14:06,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-10T18:14:06,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 754 msec 2024-11-10T18:14:06,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 761 msec 2024-11-10T18:14:06,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:06,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:14:06,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:06,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:06,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:06,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:06,187 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:06,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:06,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e6eecdc5c3e44de68bdf7a1f07049227_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:06,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741971_1147 (size=17034) 2024-11-10T18:14:06,220 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:06,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e6eecdc5c3e44de68bdf7a1f07049227_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e6eecdc5c3e44de68bdf7a1f07049227_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:06,229 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/5718aefb5b974b0484ec50abec9e6ac4, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:06,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/5718aefb5b974b0484ec50abec9e6ac4 is 175, key is test_row_0/A:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:06,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262506238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,241 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262506239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741972_1148 (size=48139) 2024-11-10T18:14:06,264 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/5718aefb5b974b0484ec50abec9e6ac4 2024-11-10T18:14:06,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/cb2ab6b4d2a24761845cf7b4d4f5a50d is 50, key is test_row_0/B:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:06,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741973_1149 (size=12001) 2024-11-10T18:14:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262506345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,346 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262506354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-10T18:14:06,475 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:06,475 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-10T18:14:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T18:14:06,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:06,478 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:06,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:06,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262506555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262506564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T18:14:06,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T18:14:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/cb2ab6b4d2a24761845cf7b4d4f5a50d 2024-11-10T18:14:06,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/ec4f88b2dfa34b55b6a43784edc65fe9 is 50, key is test_row_0/C:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:06,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741974_1150 (size=12001) 2024-11-10T18:14:06,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T18:14:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T18:14:06,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262506865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262506875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:06,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T18:14:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:06,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:06,972 INFO [master/691ea56905d2:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-10T18:14:06,972 INFO [master/691ea56905d2:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-10T18:14:07,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T18:14:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:07,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:07,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:07,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262507095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262507095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,097 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262507095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,098 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,098 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T18:14:07,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/ec4f88b2dfa34b55b6a43784edc65fe9 2024-11-10T18:14:07,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/5718aefb5b974b0484ec50abec9e6ac4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4 2024-11-10T18:14:07,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4, entries=250, sequenceid=98, filesize=47.0 K 2024-11-10T18:14:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/cb2ab6b4d2a24761845cf7b4d4f5a50d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/cb2ab6b4d2a24761845cf7b4d4f5a50d 2024-11-10T18:14:07,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/cb2ab6b4d2a24761845cf7b4d4f5a50d, entries=150, sequenceid=98, filesize=11.7 K 2024-11-10T18:14:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/ec4f88b2dfa34b55b6a43784edc65fe9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/ec4f88b2dfa34b55b6a43784edc65fe9 2024-11-10T18:14:07,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/ec4f88b2dfa34b55b6a43784edc65fe9, entries=150, sequenceid=98, filesize=11.7 K 2024-11-10T18:14:07,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a31b277868914d792297968bfe99e4fc in 958ms, sequenceid=98, compaction requested=true 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:07,145 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:07,145 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:07,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:07,146 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:07,146 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:07,146 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:07,147 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:07,147 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,147 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,147 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/aa8abbac578a46958f3c1f42f884acb0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=107.6 K 2024-11-10T18:14:07,147 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,147 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d38200c0a6f34347855094d252c7271e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/93126532b8c544eaab6f7281268d8428, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/cb2ab6b4d2a24761845cf7b4d4f5a50d] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=35.3 K 2024-11-10T18:14:07,147 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/aa8abbac578a46958f3c1f42f884acb0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4] 2024-11-10T18:14:07,147 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d38200c0a6f34347855094d252c7271e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731262444499 2024-11-10T18:14:07,147 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8abbac578a46958f3c1f42f884acb0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731262444499 2024-11-10T18:14:07,148 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 93126532b8c544eaab6f7281268d8428, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731262444879 2024-11-10T18:14:07,148 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be0bc7f297a4d47b8b4a6f7ebb3e92e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731262444879 2024-11-10T18:14:07,148 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2ab6b4d2a24761845cf7b4d4f5a50d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731262445541 2024-11-10T18:14:07,148 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5718aefb5b974b0484ec50abec9e6ac4, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731262445528 2024-11-10T18:14:07,161 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:07,163 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:07,163 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d87a1acdae0945a3ae23b9ee2124347d is 50, key is test_row_0/B:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:07,165 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111071b2240486384af89254370eed743827_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:07,168 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111071b2240486384af89254370eed743827_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:07,169 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111071b2240486384af89254370eed743827_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:07,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741975_1151 (size=12207) 2024-11-10T18:14:07,180 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d87a1acdae0945a3ae23b9ee2124347d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d87a1acdae0945a3ae23b9ee2124347d 2024-11-10T18:14:07,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741976_1152 (size=4469) 2024-11-10T18:14:07,183 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#130 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:07,184 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/9797924d71cf4e3ebfe3e71d94c4de98 is 175, key is test_row_0/A:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:07,188 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into d87a1acdae0945a3ae23b9ee2124347d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:07,188 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:07,188 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=13, startTime=1731262447145; duration=0sec 2024-11-10T18:14:07,188 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:07,189 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:07,189 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:07,190 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:07,190 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:07,190 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,190 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/cf0c63152cf249b58d1b116503a37a1d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d86faa4983842b589ae3103834e2b8e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/ec4f88b2dfa34b55b6a43784edc65fe9] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=35.3 K 2024-11-10T18:14:07,190 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cf0c63152cf249b58d1b116503a37a1d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731262444499 2024-11-10T18:14:07,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741977_1153 (size=31161) 2024-11-10T18:14:07,192 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d86faa4983842b589ae3103834e2b8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731262444879 2024-11-10T18:14:07,193 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ec4f88b2dfa34b55b6a43784edc65fe9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731262445541 2024-11-10T18:14:07,201 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/9797924d71cf4e3ebfe3e71d94c4de98 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/9797924d71cf4e3ebfe3e71d94c4de98 2024-11-10T18:14:07,206 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:07,206 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/a168481359094db299f8d1edd8abc073 is 50, key is test_row_0/C:col10/1731262445553/Put/seqid=0 2024-11-10T18:14:07,207 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into 9797924d71cf4e3ebfe3e71d94c4de98(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:07,207 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:07,207 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=13, startTime=1731262447145; duration=0sec 2024-11-10T18:14:07,207 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:07,207 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:07,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741978_1154 (size=12207) 2024-11-10T18:14:07,221 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/a168481359094db299f8d1edd8abc073 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/a168481359094db299f8d1edd8abc073 2024-11-10T18:14:07,229 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into a168481359094db299f8d1edd8abc073(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:07,230 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:07,230 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=13, startTime=1731262447145; duration=0sec 2024-11-10T18:14:07,230 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:07,230 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:07,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-10T18:14:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:07,245 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T18:14:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:07,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:07,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:07,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102050fd62bed4405c8ce773940b8ae2e0_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262446200/Put/seqid=0 2024-11-10T18:14:07,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741979_1155 (size=12154) 2024-11-10T18:14:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:07,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262507402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262507403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262507515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262507515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T18:14:07,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:07,666 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102050fd62bed4405c8ce773940b8ae2e0_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102050fd62bed4405c8ce773940b8ae2e0_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d3a0fdb20ca94c038dd751ede0201fc8, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d3a0fdb20ca94c038dd751ede0201fc8 is 175, key is test_row_0/A:col10/1731262446200/Put/seqid=0 2024-11-10T18:14:07,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741980_1156 (size=30955) 2024-11-10T18:14:07,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262507725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,729 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:07,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262507735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:07,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262508034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262508044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,073 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d3a0fdb20ca94c038dd751ede0201fc8 2024-11-10T18:14:08,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d9cc61137a6648e18619673ddd0c62f4 is 50, key is test_row_0/B:col10/1731262446200/Put/seqid=0 2024-11-10T18:14:08,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741981_1157 (size=12001) 2024-11-10T18:14:08,493 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d9cc61137a6648e18619673ddd0c62f4 2024-11-10T18:14:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/d5542250559d45eb90338f159a3aad58 is 50, key is test_row_0/C:col10/1731262446200/Put/seqid=0 2024-11-10T18:14:08,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741982_1158 (size=12001) 2024-11-10T18:14:08,508 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/d5542250559d45eb90338f159a3aad58 2024-11-10T18:14:08,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d3a0fdb20ca94c038dd751ede0201fc8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8 2024-11-10T18:14:08,520 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8, entries=150, sequenceid=119, filesize=30.2 K 2024-11-10T18:14:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d9cc61137a6648e18619673ddd0c62f4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d9cc61137a6648e18619673ddd0c62f4 2024-11-10T18:14:08,526 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d9cc61137a6648e18619673ddd0c62f4, entries=150, sequenceid=119, filesize=11.7 K 2024-11-10T18:14:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/d5542250559d45eb90338f159a3aad58 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/d5542250559d45eb90338f159a3aad58 2024-11-10T18:14:08,533 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/d5542250559d45eb90338f159a3aad58, entries=150, sequenceid=119, filesize=11.7 K 2024-11-10T18:14:08,534 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a31b277868914d792297968bfe99e4fc in 1289ms, sequenceid=119, compaction requested=false 2024-11-10T18:14:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-10T18:14:08,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-10T18:14:08,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-10T18:14:08,538 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0580 sec 2024-11-10T18:14:08,541 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.0640 sec 2024-11-10T18:14:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:08,546 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T18:14:08,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:08,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:08,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:08,546 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:08,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:08,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bf58244a7ff04f2c809707d28153e964_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:08,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741983_1159 (size=14794) 2024-11-10T18:14:08,577 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:08,584 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bf58244a7ff04f2c809707d28153e964_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bf58244a7ff04f2c809707d28153e964_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:08,586 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0dbcd6a6dd294247b3b6415206f3e846, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:08,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0dbcd6a6dd294247b3b6415206f3e846 is 175, key is test_row_0/A:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:08,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741984_1160 (size=39749) 2024-11-10T18:14:08,598 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0dbcd6a6dd294247b3b6415206f3e846 2024-11-10T18:14:08,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262508600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262508602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3d2813db46ff4b129b1e17de02914207 is 50, key is test_row_0/B:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:08,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-10T18:14:08,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741985_1161 (size=12151) 2024-11-10T18:14:08,625 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:08,626 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:08,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-10T18:14:08,628 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:08,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:08,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3d2813db46ff4b129b1e17de02914207 2024-11-10T18:14:08,629 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:08,629 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:08,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/93185440af0d4b5abd8a5559a7dcc456 is 50, key is test_row_0/C:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:08,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741986_1162 (size=12151) 2024-11-10T18:14:08,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/93185440af0d4b5abd8a5559a7dcc456 2024-11-10T18:14:08,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0dbcd6a6dd294247b3b6415206f3e846 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846 2024-11-10T18:14:08,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846, entries=200, sequenceid=134, filesize=38.8 K 2024-11-10T18:14:08,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3d2813db46ff4b129b1e17de02914207 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d2813db46ff4b129b1e17de02914207 2024-11-10T18:14:08,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d2813db46ff4b129b1e17de02914207, entries=150, sequenceid=134, filesize=11.9 K 2024-11-10T18:14:08,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/93185440af0d4b5abd8a5559a7dcc456 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/93185440af0d4b5abd8a5559a7dcc456 2024-11-10T18:14:08,688 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/93185440af0d4b5abd8a5559a7dcc456, entries=150, sequenceid=134, filesize=11.9 K 2024-11-10T18:14:08,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for a31b277868914d792297968bfe99e4fc in 143ms, sequenceid=134, compaction requested=true 2024-11-10T18:14:08,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:08,690 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:08,690 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:08,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:08,692 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101865 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:08,692 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:08,692 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:08,692 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:08,692 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,692 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,692 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d87a1acdae0945a3ae23b9ee2124347d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d9cc61137a6648e18619673ddd0c62f4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d2813db46ff4b129b1e17de02914207] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=35.5 K 2024-11-10T18:14:08,692 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/9797924d71cf4e3ebfe3e71d94c4de98, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=99.5 K 2024-11-10T18:14:08,693 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,693 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/9797924d71cf4e3ebfe3e71d94c4de98, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846] 2024-11-10T18:14:08,693 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d87a1acdae0945a3ae23b9ee2124347d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731262445541 2024-11-10T18:14:08,693 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9797924d71cf4e3ebfe3e71d94c4de98, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731262445541 2024-11-10T18:14:08,694 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d9cc61137a6648e18619673ddd0c62f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731262446200 2024-11-10T18:14:08,694 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3a0fdb20ca94c038dd751ede0201fc8, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731262446200 2024-11-10T18:14:08,694 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2813db46ff4b129b1e17de02914207, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262447398 2024-11-10T18:14:08,694 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dbcd6a6dd294247b3b6415206f3e846, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262447398 2024-11-10T18:14:08,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T18:14:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:08,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:08,720 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:08,723 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#138 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:08,724 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/455c0dbf9331468e928e04e0ed931afa is 50, key is test_row_0/B:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:08,730 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411103297bc03481e4743a289210221b7b689_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:08,733 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411103297bc03481e4743a289210221b7b689_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:08,733 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411103297bc03481e4743a289210221b7b689_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:08,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:08,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109a03349e755944e2bb86e275d8b887e4_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262448599/Put/seqid=0 2024-11-10T18:14:08,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741987_1163 (size=12459) 2024-11-10T18:14:08,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262508751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262508753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,763 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/455c0dbf9331468e928e04e0ed931afa as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/455c0dbf9331468e928e04e0ed931afa 2024-11-10T18:14:08,771 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into 455c0dbf9331468e928e04e0ed931afa(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:08,771 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:08,771 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=13, startTime=1731262448690; duration=0sec 2024-11-10T18:14:08,771 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:08,773 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:08,773 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:08,778 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:08,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741989_1165 (size=14794) 2024-11-10T18:14:08,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741988_1164 (size=4469) 2024-11-10T18:14:08,780 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:08,780 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,781 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/a168481359094db299f8d1edd8abc073, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/d5542250559d45eb90338f159a3aad58, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/93185440af0d4b5abd8a5559a7dcc456] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=35.5 K 2024-11-10T18:14:08,781 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a168481359094db299f8d1edd8abc073, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731262445541 2024-11-10T18:14:08,782 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d5542250559d45eb90338f159a3aad58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731262446200 2024-11-10T18:14:08,783 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 93185440af0d4b5abd8a5559a7dcc456, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262447398 2024-11-10T18:14:08,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:08,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:08,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:08,793 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:08,793 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bc4bbba8cafa4046a56cade6960165d3 is 50, key is test_row_0/C:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:08,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741990_1166 (size=12459) 2024-11-10T18:14:08,813 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bc4bbba8cafa4046a56cade6960165d3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bc4bbba8cafa4046a56cade6960165d3 2024-11-10T18:14:08,822 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into bc4bbba8cafa4046a56cade6960165d3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:08,822 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:08,822 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=13, startTime=1731262448690; duration=0sec 2024-11-10T18:14:08,822 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:08,822 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:08,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262508865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262508865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:08,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:08,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:08,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:08,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:08,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:09,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262509075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262509075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:09,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262509124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262509124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,128 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:09,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262509125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,128 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,129 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:09,181 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#139 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:09,182 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d3467415f2a446638c3c442a2dc67beb is 175, key is test_row_0/A:col10/1731262447398/Put/seqid=0 2024-11-10T18:14:09,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411109a03349e755944e2bb86e275d8b887e4_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a03349e755944e2bb86e275d8b887e4_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:09,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741991_1167 (size=31413) 2024-11-10T18:14:09,187 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/141ab9406b4e49ee9ca68a69e3d0d936, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:09,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/141ab9406b4e49ee9ca68a69e3d0d936 is 175, key is test_row_0/A:col10/1731262448599/Put/seqid=0 2024-11-10T18:14:09,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741992_1168 (size=39749) 2024-11-10T18:14:09,192 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/141ab9406b4e49ee9ca68a69e3d0d936 2024-11-10T18:14:09,192 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/d3467415f2a446638c3c442a2dc67beb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3467415f2a446638c3c442a2dc67beb 2024-11-10T18:14:09,198 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into d3467415f2a446638c3c442a2dc67beb(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:09,198 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:09,198 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=13, startTime=1731262448690; duration=0sec 2024-11-10T18:14:09,198 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:09,198 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:09,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/b41a11638e794326ab478a974edf455b is 50, key is test_row_0/B:col10/1731262448599/Put/seqid=0 2024-11-10T18:14:09,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741993_1169 (size=12151) 2024-11-10T18:14:09,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:09,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262509385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262509385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:09,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:09,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/b41a11638e794326ab478a974edf455b 2024-11-10T18:14:09,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/1a765d5a23f9450e8093a44c17a7cb3f is 50, key is test_row_0/C:col10/1731262448599/Put/seqid=0 2024-11-10T18:14:09,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741994_1170 (size=12151) 2024-11-10T18:14:09,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:09,857 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:09,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:09,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262509895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262509895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:09,896 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:10,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:10,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:10,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:10,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:10,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:10,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/1a765d5a23f9450e8093a44c17a7cb3f 2024-11-10T18:14:10,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/141ab9406b4e49ee9ca68a69e3d0d936 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936 2024-11-10T18:14:10,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936, entries=200, sequenceid=153, filesize=38.8 K 2024-11-10T18:14:10,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/b41a11638e794326ab478a974edf455b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b41a11638e794326ab478a974edf455b 2024-11-10T18:14:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b41a11638e794326ab478a974edf455b, entries=150, sequenceid=153, filesize=11.9 K 2024-11-10T18:14:10,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/1a765d5a23f9450e8093a44c17a7cb3f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/1a765d5a23f9450e8093a44c17a7cb3f 2024-11-10T18:14:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/1a765d5a23f9450e8093a44c17a7cb3f, entries=150, sequenceid=153, filesize=11.9 K 2024-11-10T18:14:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for a31b277868914d792297968bfe99e4fc in 1347ms, sequenceid=153, compaction requested=false 2024-11-10T18:14:10,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-10T18:14:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:10,164 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:14:10,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:10,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff19984adbde4a47aa2f8805c068b25c_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262448749/Put/seqid=0 2024-11-10T18:14:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741995_1171 (size=9814) 2024-11-10T18:14:10,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,598 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ff19984adbde4a47aa2f8805c068b25c_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff19984adbde4a47aa2f8805c068b25c_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bf49b007c2dd474e99bef950d038a48d, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:10,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bf49b007c2dd474e99bef950d038a48d is 175, key is test_row_0/A:col10/1731262448749/Put/seqid=0 2024-11-10T18:14:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741996_1172 (size=22461) 2024-11-10T18:14:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:10,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:10,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:10,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:10,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262510958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:10,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:10,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:10,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262510963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:10,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,007 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bf49b007c2dd474e99bef950d038a48d 2024-11-10T18:14:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/c71894cc6bb644ccbfe9c3b29665f5a2 is 50, key is test_row_0/B:col10/1731262448749/Put/seqid=0 2024-11-10T18:14:11,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741997_1173 (size=9757) 2024-11-10T18:14:11,033 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/c71894cc6bb644ccbfe9c3b29665f5a2 2024-11-10T18:14:11,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f55bb38e8e3f431db87ab4d0bb2f7ff9 is 50, key is test_row_0/C:col10/1731262448749/Put/seqid=0 2024-11-10T18:14:11,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741998_1174 (size=9757) 2024-11-10T18:14:11,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262511075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262511075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262511285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262511294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,450 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f55bb38e8e3f431db87ab4d0bb2f7ff9 2024-11-10T18:14:11,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bf49b007c2dd474e99bef950d038a48d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d 2024-11-10T18:14:11,460 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d, entries=100, sequenceid=169, filesize=21.9 K 2024-11-10T18:14:11,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/c71894cc6bb644ccbfe9c3b29665f5a2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/c71894cc6bb644ccbfe9c3b29665f5a2 2024-11-10T18:14:11,466 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/c71894cc6bb644ccbfe9c3b29665f5a2, entries=100, sequenceid=169, filesize=9.5 K 2024-11-10T18:14:11,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f55bb38e8e3f431db87ab4d0bb2f7ff9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f55bb38e8e3f431db87ab4d0bb2f7ff9 2024-11-10T18:14:11,472 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f55bb38e8e3f431db87ab4d0bb2f7ff9, entries=100, sequenceid=169, filesize=9.5 K 2024-11-10T18:14:11,473 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a31b277868914d792297968bfe99e4fc in 1309ms, sequenceid=169, compaction requested=true 2024-11-10T18:14:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:11,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-10T18:14:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-10T18:14:11,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-10T18:14:11,476 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8460 sec 2024-11-10T18:14:11,478 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.8510 sec 2024-11-10T18:14:11,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:11,596 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:14:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:11,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:11,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101d73b46f86724a148564803568b15513_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:11,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741999_1175 (size=14794) 2024-11-10T18:14:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262511625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262511626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262511735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262511745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,753 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262511945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:11,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:11,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262511965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:11,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:11,967 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:12,012 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:12,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101d73b46f86724a148564803568b15513_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101d73b46f86724a148564803568b15513_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:12,018 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/694571a79ded49f78a1073dff544ca01, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:12,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/694571a79ded49f78a1073dff544ca01 is 175, key is test_row_0/A:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:12,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742000_1176 (size=39749) 2024-11-10T18:14:12,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:12,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262512254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:12,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262512275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:12,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:12,424 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/694571a79ded49f78a1073dff544ca01 2024-11-10T18:14:12,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/09edf518650f4c50b8e75d2fd53f6d33 is 50, key is test_row_0/B:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:12,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742001_1177 (size=12151) 2024-11-10T18:14:12,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262512764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:12,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262512785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:12,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-10T18:14:12,795 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:12,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-10T18:14:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T18:14:12,798 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:12,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:12,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:12,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/09edf518650f4c50b8e75d2fd53f6d33 2024-11-10T18:14:12,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/36aed4e35e0946c1b8d583c3b9396f94 is 50, key is test_row_0/C:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:12,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742002_1178 (size=12151) 2024-11-10T18:14:12,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/36aed4e35e0946c1b8d583c3b9396f94 2024-11-10T18:14:12,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/694571a79ded49f78a1073dff544ca01 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01 2024-11-10T18:14:12,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01, entries=200, sequenceid=189, filesize=38.8 K 2024-11-10T18:14:12,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/09edf518650f4c50b8e75d2fd53f6d33 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/09edf518650f4c50b8e75d2fd53f6d33 2024-11-10T18:14:12,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/09edf518650f4c50b8e75d2fd53f6d33, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T18:14:12,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/36aed4e35e0946c1b8d583c3b9396f94 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/36aed4e35e0946c1b8d583c3b9396f94 2024-11-10T18:14:12,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/36aed4e35e0946c1b8d583c3b9396f94, entries=150, sequenceid=189, filesize=11.9 K 2024-11-10T18:14:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for a31b277868914d792297968bfe99e4fc in 1278ms, sequenceid=189, compaction requested=true 2024-11-10T18:14:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:12,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:12,875 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:12,875 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:12,875 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:12,876 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133372 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:12,876 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:12,876 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:12,876 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:12,876 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:12,876 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:12,876 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/455c0dbf9331468e928e04e0ed931afa, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b41a11638e794326ab478a974edf455b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/c71894cc6bb644ccbfe9c3b29665f5a2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/09edf518650f4c50b8e75d2fd53f6d33] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=45.4 K 2024-11-10T18:14:12,876 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3467415f2a446638c3c442a2dc67beb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=130.2 K 2024-11-10T18:14:12,876 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:12,877 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3467415f2a446638c3c442a2dc67beb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01] 2024-11-10T18:14:12,877 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 455c0dbf9331468e928e04e0ed931afa, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262447398 2024-11-10T18:14:12,877 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3467415f2a446638c3c442a2dc67beb, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262447398 2024-11-10T18:14:12,877 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b41a11638e794326ab478a974edf455b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731262448597 2024-11-10T18:14:12,878 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 141ab9406b4e49ee9ca68a69e3d0d936, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731262448590 2024-11-10T18:14:12,878 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c71894cc6bb644ccbfe9c3b29665f5a2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731262448749 2024-11-10T18:14:12,878 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf49b007c2dd474e99bef950d038a48d, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731262448749 2024-11-10T18:14:12,878 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 09edf518650f4c50b8e75d2fd53f6d33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731262450951 2024-11-10T18:14:12,878 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 694571a79ded49f78a1073dff544ca01, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731262450951 2024-11-10T18:14:12,892 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:12,893 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e1ccbde3512e49189dc626e4458fcbc0 is 50, key is test_row_0/B:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:12,899 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T18:14:12,909 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100a6d0a004b6d474bb48b6672305c10ce_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:12,912 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100a6d0a004b6d474bb48b6672305c10ce_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:12,912 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100a6d0a004b6d474bb48b6672305c10ce_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:12,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742003_1179 (size=12595) 2024-11-10T18:14:12,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742004_1180 (size=4469) 2024-11-10T18:14:12,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-10T18:14:12,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:12,951 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:12,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:12,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107d4c3957dbd441189a61deab666f4193_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262451622/Put/seqid=0 2024-11-10T18:14:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742005_1181 (size=12304) 2024-11-10T18:14:13,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T18:14:13,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:13,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:13,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262513172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262513173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262513176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262513285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262513285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262513294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,327 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#151 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:13,327 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bb5ac4985f074fc4a32b89f9ea08eb9c is 175, key is test_row_0/A:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:13,328 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e1ccbde3512e49189dc626e4458fcbc0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e1ccbde3512e49189dc626e4458fcbc0 2024-11-10T18:14:13,334 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into e1ccbde3512e49189dc626e4458fcbc0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:13,334 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:13,334 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=12, startTime=1731262452874; duration=0sec 2024-11-10T18:14:13,334 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:13,335 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:13,335 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:13,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742006_1182 (size=31549) 2024-11-10T18:14:13,337 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46518 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:13,337 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:13,338 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:13,338 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bc4bbba8cafa4046a56cade6960165d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/1a765d5a23f9450e8093a44c17a7cb3f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f55bb38e8e3f431db87ab4d0bb2f7ff9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/36aed4e35e0946c1b8d583c3b9396f94] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=45.4 K 2024-11-10T18:14:13,338 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting bc4bbba8cafa4046a56cade6960165d3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262447398 2024-11-10T18:14:13,339 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a765d5a23f9450e8093a44c17a7cb3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731262448597 2024-11-10T18:14:13,339 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f55bb38e8e3f431db87ab4d0bb2f7ff9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731262448749 2024-11-10T18:14:13,339 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 36aed4e35e0946c1b8d583c3b9396f94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731262450951 2024-11-10T18:14:13,350 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#153 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:13,350 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/3d5e1f107257468b90b119c49384b857 is 50, key is test_row_0/C:col10/1731262450951/Put/seqid=0 2024-11-10T18:14:13,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742007_1183 (size=12595) 2024-11-10T18:14:13,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:13,370 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107d4c3957dbd441189a61deab666f4193_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d4c3957dbd441189a61deab666f4193_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/26830ac412a74bba993419deaaa570dc, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:13,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/26830ac412a74bba993419deaaa570dc is 175, key is test_row_0/A:col10/1731262451622/Put/seqid=0 2024-11-10T18:14:13,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742008_1184 (size=31105) 2024-11-10T18:14:13,377 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/26830ac412a74bba993419deaaa570dc 2024-11-10T18:14:13,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3d4414daf678466a80df0d1d4fe99b92 is 50, key is test_row_0/B:col10/1731262451622/Put/seqid=0 2024-11-10T18:14:13,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742009_1185 (size=12151) 2024-11-10T18:14:13,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T18:14:13,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262513495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262513495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262513504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,742 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bb5ac4985f074fc4a32b89f9ea08eb9c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bb5ac4985f074fc4a32b89f9ea08eb9c 2024-11-10T18:14:13,747 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into bb5ac4985f074fc4a32b89f9ea08eb9c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:13,748 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:13,748 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=12, startTime=1731262452874; duration=0sec 2024-11-10T18:14:13,748 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:13,748 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:13,759 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/3d5e1f107257468b90b119c49384b857 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d5e1f107257468b90b119c49384b857 2024-11-10T18:14:13,764 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into 3d5e1f107257468b90b119c49384b857(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:13,764 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:13,764 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=12, startTime=1731262452875; duration=0sec 2024-11-10T18:14:13,764 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:13,764 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:13,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262513774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,775 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,791 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3d4414daf678466a80df0d1d4fe99b92 2024-11-10T18:14:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262513795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,796 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/b8a500b1ae2c4200870c9d4eaae2a7e6 is 50, key is test_row_0/C:col10/1731262451622/Put/seqid=0 2024-11-10T18:14:13,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742010_1186 (size=12151) 2024-11-10T18:14:13,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262513805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262513805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:13,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262513814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:13,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:13,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T18:14:14,205 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/b8a500b1ae2c4200870c9d4eaae2a7e6 2024-11-10T18:14:14,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/26830ac412a74bba993419deaaa570dc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc 2024-11-10T18:14:14,230 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc, entries=150, sequenceid=201, filesize=30.4 K 2024-11-10T18:14:14,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3d4414daf678466a80df0d1d4fe99b92 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d4414daf678466a80df0d1d4fe99b92 2024-11-10T18:14:14,236 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d4414daf678466a80df0d1d4fe99b92, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T18:14:14,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/b8a500b1ae2c4200870c9d4eaae2a7e6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/b8a500b1ae2c4200870c9d4eaae2a7e6 2024-11-10T18:14:14,241 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/b8a500b1ae2c4200870c9d4eaae2a7e6, entries=150, sequenceid=201, filesize=11.9 K 2024-11-10T18:14:14,242 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a31b277868914d792297968bfe99e4fc in 1291ms, sequenceid=201, compaction requested=false 2024-11-10T18:14:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:14,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-10T18:14:14,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-10T18:14:14,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-10T18:14:14,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4450 sec 2024-11-10T18:14:14,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.4500 sec 2024-11-10T18:14:14,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:14,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-10T18:14:14,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262453163, max timestamp=1731262454315Num uniques 859; 2024-11-10T18:14:14,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262453163, max timestamp=1731262454315Num uniques 859; 2024-11-10T18:14:14,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262453163, max timestamp=1731262454315Num uniques 859; 2024-11-10T18:14:14,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:14,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:14,320 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:14,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:14,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:14,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:14,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:14,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:14,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:14,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107caa5409b75c482d8aa56f6f3a36827f_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:14,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742011_1187 (size=17284) 2024-11-10T18:14:14,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262454344, max timestamp=1731262454358Num uniques 859; 2024-11-10T18:14:14,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262454344, max timestamp=1731262454358Num uniques 859; 2024-11-10T18:14:14,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:14:14,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:14:14,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262454344, max timestamp=1731262454358Num uniques 859; 2024-11-10T18:14:14,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:14:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262514419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262514420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262514429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,436 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262514535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262514535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262514545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,738 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:14,743 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107caa5409b75c482d8aa56f6f3a36827f_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107caa5409b75c482d8aa56f6f3a36827f_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:14,744 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0b8c4b41833e4792a4cdfafbc94aea91, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:14,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0b8c4b41833e4792a4cdfafbc94aea91 is 175, key is test_row_0/A:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:14,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262514745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,746 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262514745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742012_1188 (size=48389) 2024-11-10T18:14:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262514754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:14,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-10T18:14:14,945 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:14,945 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-10T18:14:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T18:14:14,947 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:14,948 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:14,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T18:14:15,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262515055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262515055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262515064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0b8c4b41833e4792a4cdfafbc94aea91 2024-11-10T18:14:15,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/597501df797a41d8b52cda88c7b47fe3 is 50, key is test_row_0/B:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:15,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742013_1189 (size=12151) 2024-11-10T18:14:15,254 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,254 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T18:14:15,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:15,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262515565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262515565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/597501df797a41d8b52cda88c7b47fe3 2024-11-10T18:14:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T18:14:15,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262515574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f4d8fe86cbae484c860151467f98c388 is 50, key is test_row_0/C:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:15,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742014_1190 (size=12151) 2024-11-10T18:14:15,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:15,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262515794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,802 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:15,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262515815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:15,821 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:15,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:15,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:15,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:15,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:15,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:15,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f4d8fe86cbae484c860151467f98c388 2024-11-10T18:14:15,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0b8c4b41833e4792a4cdfafbc94aea91 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91 2024-11-10T18:14:15,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91, entries=250, sequenceid=226, filesize=47.3 K 2024-11-10T18:14:15,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/597501df797a41d8b52cda88c7b47fe3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/597501df797a41d8b52cda88c7b47fe3 2024-11-10T18:14:16,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/597501df797a41d8b52cda88c7b47fe3, entries=150, sequenceid=226, filesize=11.9 K 2024-11-10T18:14:16,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f4d8fe86cbae484c860151467f98c388 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f4d8fe86cbae484c860151467f98c388 2024-11-10T18:14:16,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f4d8fe86cbae484c860151467f98c388, entries=150, sequenceid=226, filesize=11.9 K 2024-11-10T18:14:16,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for a31b277868914d792297968bfe99e4fc in 1690ms, sequenceid=226, compaction requested=true 2024-11-10T18:14:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:16,008 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:16,008 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:16,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:16,009 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:16,009 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111043 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:16,009 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:16,009 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:16,009 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:16,010 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bb5ac4985f074fc4a32b89f9ea08eb9c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=108.4 K 2024-11-10T18:14:16,010 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:16,010 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:16,010 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:16,010 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bb5ac4985f074fc4a32b89f9ea08eb9c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91] 2024-11-10T18:14:16,010 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e1ccbde3512e49189dc626e4458fcbc0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d4414daf678466a80df0d1d4fe99b92, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/597501df797a41d8b52cda88c7b47fe3] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=36.0 K 2024-11-10T18:14:16,010 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb5ac4985f074fc4a32b89f9ea08eb9c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731262450951 2024-11-10T18:14:16,010 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e1ccbde3512e49189dc626e4458fcbc0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731262450951 2024-11-10T18:14:16,011 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26830ac412a74bba993419deaaa570dc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731262451622 2024-11-10T18:14:16,011 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d4414daf678466a80df0d1d4fe99b92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731262451622 2024-11-10T18:14:16,011 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b8c4b41833e4792a4cdfafbc94aea91, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731262453163 2024-11-10T18:14:16,011 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 597501df797a41d8b52cda88c7b47fe3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731262453171 2024-11-10T18:14:16,018 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:16,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-10T18:14:16,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:16,021 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=154.31 KB heapSize=430.69 KB 2024-11-10T18:14:16,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:16,023 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#160 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:16,023 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d6e405051c6f4e26800c3a5576df61b1 is 50, key is test_row_0/B:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:16,024 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100c2d7d30260b417387577117d3bc8b4b_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:16,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:16,026 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100c2d7d30260b417387577117d3bc8b4b_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:16,027 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100c2d7d30260b417387577117d3bc8b4b_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:16,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742015_1191 (size=12697) 2024-11-10T18:14:16,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742016_1192 (size=4469) 2024-11-10T18:14:16,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111022edc569759a44e48e7a4eb041713c01_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262454392/Put/seqid=0 2024-11-10T18:14:16,047 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/d6e405051c6f4e26800c3a5576df61b1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d6e405051c6f4e26800c3a5576df61b1 2024-11-10T18:14:16,054 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into d6e405051c6f4e26800c3a5576df61b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:16,054 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:16,054 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=13, startTime=1731262456008; duration=0sec 2024-11-10T18:14:16,054 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:16,054 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:16,054 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:16,056 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:16,056 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:16,056 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:16,056 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d5e1f107257468b90b119c49384b857, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/b8a500b1ae2c4200870c9d4eaae2a7e6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f4d8fe86cbae484c860151467f98c388] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=36.0 K 2024-11-10T18:14:16,057 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d5e1f107257468b90b119c49384b857, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731262450951 2024-11-10T18:14:16,057 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b8a500b1ae2c4200870c9d4eaae2a7e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731262451622 2024-11-10T18:14:16,058 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f4d8fe86cbae484c860151467f98c388, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731262453171 2024-11-10T18:14:16,075 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:16,075 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/47b347e4a20346ada99dfba6ab221c20 is 50, key is test_row_0/C:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:16,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742017_1193 (size=12454) 2024-11-10T18:14:16,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742018_1194 (size=12697) 2024-11-10T18:14:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T18:14:16,091 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/47b347e4a20346ada99dfba6ab221c20 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/47b347e4a20346ada99dfba6ab221c20 2024-11-10T18:14:16,099 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into 47b347e4a20346ada99dfba6ab221c20(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:16,099 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:16,100 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=13, startTime=1731262456009; duration=0sec 2024-11-10T18:14:16,100 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:16,100 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:16,441 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#159 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:16,442 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/34c6cb79ae104479875a510c708eb496 is 175, key is test_row_0/A:col10/1731262453171/Put/seqid=0 2024-11-10T18:14:16,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742019_1195 (size=31651) 2024-11-10T18:14:16,453 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/34c6cb79ae104479875a510c708eb496 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/34c6cb79ae104479875a510c708eb496 2024-11-10T18:14:16,460 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into 34c6cb79ae104479875a510c708eb496(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:16,460 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:16,460 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=13, startTime=1731262456008; duration=0sec 2024-11-10T18:14:16,460 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:16,460 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:16,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,480 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111022edc569759a44e48e7a4eb041713c01_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111022edc569759a44e48e7a4eb041713c01_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:16,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/b75e395163f24e05b40fe6360cd16d24, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:16,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/b75e395163f24e05b40fe6360cd16d24 is 175, key is test_row_0/A:col10/1731262454392/Put/seqid=0 2024-11-10T18:14:16,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742020_1196 (size=31255) 2024-11-10T18:14:16,489 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=263, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/b75e395163f24e05b40fe6360cd16d24 2024-11-10T18:14:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/b353ae1d77fe48da8370bf5036962f0d is 50, key is test_row_0/B:col10/1731262454392/Put/seqid=0 2024-11-10T18:14:16,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742021_1197 (size=12301) 2024-11-10T18:14:16,504 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/b353ae1d77fe48da8370bf5036962f0d 2024-11-10T18:14:16,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/124fbbf766a743578ee6bde38db6c4d3 is 50, key is test_row_0/C:col10/1731262454392/Put/seqid=0 2024-11-10T18:14:16,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742022_1198 (size=12301) 2024-11-10T18:14:16,520 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/124fbbf766a743578ee6bde38db6c4d3 2024-11-10T18:14:16,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/b75e395163f24e05b40fe6360cd16d24 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24 2024-11-10T18:14:16,532 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24, entries=150, sequenceid=263, filesize=30.5 K 2024-11-10T18:14:16,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/b353ae1d77fe48da8370bf5036962f0d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b353ae1d77fe48da8370bf5036962f0d 2024-11-10T18:14:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,538 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b353ae1d77fe48da8370bf5036962f0d, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T18:14:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/124fbbf766a743578ee6bde38db6c4d3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/124fbbf766a743578ee6bde38db6c4d3 2024-11-10T18:14:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,544 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/124fbbf766a743578ee6bde38db6c4d3, entries=150, sequenceid=263, filesize=12.0 K 2024-11-10T18:14:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,545 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~430.64 KB/440976, currentSize=0 B/0 for a31b277868914d792297968bfe99e4fc in 524ms, sequenceid=263, compaction requested=false 2024-11-10T18:14:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:16,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:16,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-10T18:14:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-10T18:14:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-10T18:14:16,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5990 sec 2024-11-10T18:14:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.6050 sec 2024-11-10T18:14:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:16,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:16,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:16,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:16,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:16,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101da97df86d0f4f2c82b3cc0627dd84d4_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:16,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742023_1199 (size=22618) 2024-11-10T18:14:16,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731262456623, max timestamp=1731262456681Num uniques 859; 2024-11-10T18:14:16,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731262456623, max timestamp=1731262456681Num uniques 859; 2024-11-10T18:14:16,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T18:14:16,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T18:14:16,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=859, cellSize=39335, totalHeapSize=89576, min timestamp=1731262456623, max timestamp=1731262456681Num uniques 859; 2024-11-10T18:14:16,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=39335, suffix heap size=117064,new segment heap size=89576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=859 2024-11-10T18:14:16,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:16,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262516693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262516694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262516697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:16,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:16,707 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:16,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262516815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:16,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262516815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:16,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:16,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262516815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:16,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262517025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262517025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262517025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:17,053 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101da97df86d0f4f2c82b3cc0627dd84d4_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101da97df86d0f4f2c82b3cc0627dd84d4_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:17,054 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/352c84dff5d44062bc008293c1adf5c9, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:17,055 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/352c84dff5d44062bc008293c1adf5c9 is 175, key is test_row_0/A:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:17,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742024_1200 (size=66019) 2024-11-10T18:14:17,069 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/352c84dff5d44062bc008293c1adf5c9 2024-11-10T18:14:17,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/9b6eaef1b6b94e058012c9fee45322cd is 50, key is test_row_0/B:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:17,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742025_1201 (size=9857) 2024-11-10T18:14:17,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/9b6eaef1b6b94e058012c9fee45322cd 2024-11-10T18:14:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-10T18:14:17,096 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:17,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-10T18:14:17,100 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:17,101 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:17,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:17,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T18:14:17,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/c177ac0d4aa64849adb26894232dfd16 is 50, key is test_row_0/C:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:17,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742026_1202 (size=9857) 2024-11-10T18:14:17,127 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/c177ac0d4aa64849adb26894232dfd16 2024-11-10T18:14:17,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/352c84dff5d44062bc008293c1adf5c9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9 2024-11-10T18:14:17,138 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9, entries=350, sequenceid=276, filesize=64.5 K 2024-11-10T18:14:17,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/9b6eaef1b6b94e058012c9fee45322cd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/9b6eaef1b6b94e058012c9fee45322cd 2024-11-10T18:14:17,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/9b6eaef1b6b94e058012c9fee45322cd, entries=100, sequenceid=276, filesize=9.6 K 2024-11-10T18:14:17,144 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-10T18:14:17,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/c177ac0d4aa64849adb26894232dfd16 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c177ac0d4aa64849adb26894232dfd16 2024-11-10T18:14:17,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c177ac0d4aa64849adb26894232dfd16, entries=100, sequenceid=276, filesize=9.6 K 2024-11-10T18:14:17,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=154.31 KB/158010 for a31b277868914d792297968bfe99e4fc in 530ms, sequenceid=276, compaction requested=true 2024-11-10T18:14:17,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:17,153 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:17,153 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:17,153 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:17,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:17,154 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:17,154 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:17,154 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:17,154 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:17,154 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/34c6cb79ae104479875a510c708eb496, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=125.9 K 2024-11-10T18:14:17,154 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:17,154 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/34c6cb79ae104479875a510c708eb496, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9] 2024-11-10T18:14:17,155 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34c6cb79ae104479875a510c708eb496, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731262453171 2024-11-10T18:14:17,155 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b75e395163f24e05b40fe6360cd16d24, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262454384 2024-11-10T18:14:17,155 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:17,155 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:17,155 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:17,155 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d6e405051c6f4e26800c3a5576df61b1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b353ae1d77fe48da8370bf5036962f0d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/9b6eaef1b6b94e058012c9fee45322cd] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=34.0 K 2024-11-10T18:14:17,156 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 352c84dff5d44062bc008293c1adf5c9, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731262456604 2024-11-10T18:14:17,156 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e405051c6f4e26800c3a5576df61b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731262453171 2024-11-10T18:14:17,156 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b353ae1d77fe48da8370bf5036962f0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262454384 2024-11-10T18:14:17,157 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b6eaef1b6b94e058012c9fee45322cd, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731262456615 2024-11-10T18:14:17,172 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:17,173 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/082eab1f43e4464db0ae33ecfc22862a is 50, key is test_row_0/B:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:17,174 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:17,188 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105478af93c34e4a009e45489cd420ee1d_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:17,191 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105478af93c34e4a009e45489cd420ee1d_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:17,191 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105478af93c34e4a009e45489cd420ee1d_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:17,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742027_1203 (size=12949) 2024-11-10T18:14:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742028_1204 (size=4469) 2024-11-10T18:14:17,201 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#169 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:17,202 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bcb631e460b8417da8a85c03071c4bea is 175, key is test_row_0/A:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:17,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T18:14:17,205 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/082eab1f43e4464db0ae33ecfc22862a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/082eab1f43e4464db0ae33ecfc22862a 2024-11-10T18:14:17,210 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into 082eab1f43e4464db0ae33ecfc22862a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:17,211 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:17,211 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=13, startTime=1731262457153; duration=0sec 2024-11-10T18:14:17,211 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:17,211 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:17,211 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:17,212 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:17,212 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:17,212 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:17,212 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/47b347e4a20346ada99dfba6ab221c20, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/124fbbf766a743578ee6bde38db6c4d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c177ac0d4aa64849adb26894232dfd16] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=34.0 K 2024-11-10T18:14:17,213 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 47b347e4a20346ada99dfba6ab221c20, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731262453171 2024-11-10T18:14:17,214 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 124fbbf766a743578ee6bde38db6c4d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731262454384 2024-11-10T18:14:17,215 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c177ac0d4aa64849adb26894232dfd16, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731262456615 2024-11-10T18:14:17,229 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#170 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:17,230 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/4d290dc025d247d8b0fde56861a015da is 50, key is test_row_0/C:col10/1731262456621/Put/seqid=0 2024-11-10T18:14:17,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742029_1205 (size=32010) 2024-11-10T18:14:17,239 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/bcb631e460b8417da8a85c03071c4bea as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bcb631e460b8417da8a85c03071c4bea 2024-11-10T18:14:17,245 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into bcb631e460b8417da8a85c03071c4bea(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:17,245 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:17,245 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=13, startTime=1731262457152; duration=0sec 2024-11-10T18:14:17,245 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:17,246 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:17,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742030_1206 (size=12949) 2024-11-10T18:14:17,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:17,254 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=154.31 KB heapSize=379.13 KB 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:17,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:17,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bbeab4a7a5604bd4a9dd5f310764b844_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262456691/Put/seqid=0 2024-11-10T18:14:17,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742031_1207 (size=12454) 2024-11-10T18:14:17,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:17,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:17,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262517343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262517344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262517348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T18:14:17,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262517455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262517455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262517455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,656 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/4d290dc025d247d8b0fde56861a015da as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/4d290dc025d247d8b0fde56861a015da 2024-11-10T18:14:17,662 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into 4d290dc025d247d8b0fde56861a015da(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:17,662 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:17,662 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=13, startTime=1731262457153; duration=0sec 2024-11-10T18:14:17,662 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:17,662 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262517665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262517665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262517665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:17,675 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bbeab4a7a5604bd4a9dd5f310764b844_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bbeab4a7a5604bd4a9dd5f310764b844_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:17,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0a75f73a6de641a7822e83cf7930f718, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:17,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0a75f73a6de641a7822e83cf7930f718 is 175, key is test_row_0/A:col10/1731262456691/Put/seqid=0 2024-11-10T18:14:17,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742032_1208 (size=31255) 2024-11-10T18:14:17,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T18:14:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262517988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262517988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262517988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:18,081 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0a75f73a6de641a7822e83cf7930f718 2024-11-10T18:14:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/21ef03eb30a64e799d61af2c0b26bf6b is 50, key is test_row_0/B:col10/1731262456691/Put/seqid=0 2024-11-10T18:14:18,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742033_1209 (size=12301) 2024-11-10T18:14:18,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T18:14:18,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262518494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:18,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262518494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:18,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262518494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:18,499 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/21ef03eb30a64e799d61af2c0b26bf6b 2024-11-10T18:14:18,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bf65327a4cd94affab7e37cba4999d08 is 50, key is test_row_0/C:col10/1731262456691/Put/seqid=0 2024-11-10T18:14:18,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742034_1210 (size=12301) 2024-11-10T18:14:18,913 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bf65327a4cd94affab7e37cba4999d08 2024-11-10T18:14:18,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/0a75f73a6de641a7822e83cf7930f718 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718 2024-11-10T18:14:18,922 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718, entries=150, sequenceid=304, filesize=30.5 K 2024-11-10T18:14:18,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/21ef03eb30a64e799d61af2c0b26bf6b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/21ef03eb30a64e799d61af2c0b26bf6b 2024-11-10T18:14:18,927 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/21ef03eb30a64e799d61af2c0b26bf6b, entries=150, sequenceid=304, filesize=12.0 K 2024-11-10T18:14:18,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bf65327a4cd94affab7e37cba4999d08 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bf65327a4cd94affab7e37cba4999d08 2024-11-10T18:14:18,934 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bf65327a4cd94affab7e37cba4999d08, entries=150, sequenceid=304, filesize=12.0 K 2024-11-10T18:14:18,935 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~379.08 KB/388176, currentSize=46.96 KB/48090 for a31b277868914d792297968bfe99e4fc in 1680ms, sequenceid=304, compaction requested=false 2024-11-10T18:14:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:18,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-10T18:14:18,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-10T18:14:18,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-10T18:14:18,939 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8350 sec 2024-11-10T18:14:18,941 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.8420 sec 2024-11-10T18:14:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-10T18:14:19,245 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:19,246 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-10T18:14:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T18:14:19,248 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:19,248 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:19,248 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:19,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T18:14:19,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-10T18:14:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:19,401 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:19,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:19,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:19,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101c8b08b95df04ace8fdae7f9bb439377_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262457347/Put/seqid=0 2024-11-10T18:14:19,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742035_1211 (size=12454) 2024-11-10T18:14:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:19,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731262459509, max timestamp=1731262459537Num uniques 859; 2024-11-10T18:14:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T18:14:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731262459509, max timestamp=1731262459537Num uniques 859; 2024-11-10T18:14:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T18:14:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731262459509, max timestamp=1731262459537Num uniques 859; 2024-11-10T18:14:19,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-10T18:14:19,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T18:14:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262519568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262519568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262519569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262519685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262519685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262519685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42860 deadline: 1731262519824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,826 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:19,843 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411101c8b08b95df04ace8fdae7f9bb439377_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101c8b08b95df04ace8fdae7f9bb439377_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/3aa9ba4e2dab4ebd85475a94bffa8f88, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/3aa9ba4e2dab4ebd85475a94bffa8f88 is 175, key is test_row_0/A:col10/1731262457347/Put/seqid=0 2024-11-10T18:14:19,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742036_1212 (size=31255) 2024-11-10T18:14:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42888 deadline: 1731262519855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,860 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in a31b277868914d792297968bfe99e4fc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8233 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T18:14:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262519894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262519894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262519894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:19,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:20,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262520205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:20,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262520205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:20,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:20,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262520214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,219 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:20,249 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/3aa9ba4e2dab4ebd85475a94bffa8f88 2024-11-10T18:14:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e58cf688d4f242469db5d19a882a2f5e is 50, key is test_row_0/B:col10/1731262457347/Put/seqid=0 2024-11-10T18:14:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742037_1213 (size=12301) 2024-11-10T18:14:20,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T18:14:20,492 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T18:14:20,661 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e58cf688d4f242469db5d19a882a2f5e 2024-11-10T18:14:20,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/7dad95c4462443a08fff55bf8ce39138 is 50, key is test_row_0/C:col10/1731262457347/Put/seqid=0 2024-11-10T18:14:20,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742038_1214 (size=12301) 2024-11-10T18:14:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262520715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:20,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262520715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:20,722 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262520724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,078 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/7dad95c4462443a08fff55bf8ce39138 2024-11-10T18:14:21,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/3aa9ba4e2dab4ebd85475a94bffa8f88 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88 2024-11-10T18:14:21,087 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88, entries=150, sequenceid=315, filesize=30.5 K 2024-11-10T18:14:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e58cf688d4f242469db5d19a882a2f5e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e58cf688d4f242469db5d19a882a2f5e 2024-11-10T18:14:21,091 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e58cf688d4f242469db5d19a882a2f5e, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T18:14:21,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/7dad95c4462443a08fff55bf8ce39138 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7dad95c4462443a08fff55bf8ce39138 2024-11-10T18:14:21,096 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7dad95c4462443a08fff55bf8ce39138, entries=150, sequenceid=315, filesize=12.0 K 2024-11-10T18:14:21,097 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for a31b277868914d792297968bfe99e4fc in 1696ms, sequenceid=315, compaction requested=true 2024-11-10T18:14:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:21,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-10T18:14:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-10T18:14:21,101 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-10T18:14:21,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8510 sec 2024-11-10T18:14:21,104 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8560 sec 2024-11-10T18:14:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-10T18:14:21,395 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:21,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-10T18:14:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T18:14:21,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:21,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:21,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T18:14:21,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-10T18:14:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:21,551 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-10T18:14:21,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:21,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:21,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f3ff6546b2ed467dbbed570b196193bf_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262459567/Put/seqid=0 2024-11-10T18:14:21,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742039_1215 (size=12454) 2024-11-10T18:14:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T18:14:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:21,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. as already flushing 2024-11-10T18:14:21,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262521745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:21,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262521746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,750 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262521750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:21,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262521855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:21,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262521855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:21,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262521877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:21,976 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110f3ff6546b2ed467dbbed570b196193bf_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f3ff6546b2ed467dbbed570b196193bf_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:21,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1c01948581a84e77aac1d75305f66d8f, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:21,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1c01948581a84e77aac1d75305f66d8f is 175, key is test_row_0/A:col10/1731262459567/Put/seqid=0 2024-11-10T18:14:21,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742040_1216 (size=31255) 2024-11-10T18:14:22,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T18:14:22,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262522085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262522085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262522085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,382 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=349, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1c01948581a84e77aac1d75305f66d8f 2024-11-10T18:14:22,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e6af39e7b5894d4ea706f6aff59aad0b is 50, key is test_row_0/B:col10/1731262459567/Put/seqid=0 2024-11-10T18:14:22,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742041_1217 (size=12301) 2024-11-10T18:14:22,397 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e6af39e7b5894d4ea706f6aff59aad0b 2024-11-10T18:14:22,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262522394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262522395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262522395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/7a5713f91c5343829cf5aab4109a7457 is 50, key is test_row_0/C:col10/1731262459567/Put/seqid=0 2024-11-10T18:14:22,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742042_1218 (size=12301) 2024-11-10T18:14:22,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T18:14:22,809 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/7a5713f91c5343829cf5aab4109a7457 2024-11-10T18:14:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1c01948581a84e77aac1d75305f66d8f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f 2024-11-10T18:14:22,818 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f, entries=150, sequenceid=349, filesize=30.5 K 2024-11-10T18:14:22,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/e6af39e7b5894d4ea706f6aff59aad0b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e6af39e7b5894d4ea706f6aff59aad0b 2024-11-10T18:14:22,824 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e6af39e7b5894d4ea706f6aff59aad0b, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T18:14:22,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/7a5713f91c5343829cf5aab4109a7457 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7a5713f91c5343829cf5aab4109a7457 2024-11-10T18:14:22,829 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7a5713f91c5343829cf5aab4109a7457, entries=150, sequenceid=349, filesize=12.0 K 2024-11-10T18:14:22,829 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for a31b277868914d792297968bfe99e4fc in 1278ms, sequenceid=349, compaction requested=true 2024-11-10T18:14:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:22,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-10T18:14:22,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-10T18:14:22,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-10T18:14:22,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4320 sec 2024-11-10T18:14:22,835 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.4370 sec 2024-11-10T18:14:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:22,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:14:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:22,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c0c6b8c45f9464795410996c0106f23_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:22,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742043_1219 (size=17534) 2024-11-10T18:14:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262522937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262522938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,944 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:22,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262522944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:22,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262523054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262523054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,057 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262523054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,180 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-10T18:14:23,180 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:23,180 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,180 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:23,184 INFO [Thread-649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-649. 2024-11-10T18:14:23,184 DEBUG [Thread-649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:23,184 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,184 DEBUG [Thread-649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:23,187 INFO [Thread-651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-651. 2024-11-10T18:14:23,187 DEBUG [Thread-651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:23,187 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,187 DEBUG [Thread-651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:23,191 INFO [Thread-653 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-653. 2024-11-10T18:14:23,191 DEBUG [Thread-653 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:23,191 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,191 DEBUG [Thread-653 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:23,191 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262523265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262523265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262523265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,324 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:23,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c0c6b8c45f9464795410996c0106f23_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c0c6b8c45f9464795410996c0106f23_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:23,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1a82e239115744b3ae8f9e64f40a5042, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:23,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1a82e239115744b3ae8f9e64f40a5042 is 175, key is test_row_0/A:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:23,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742044_1220 (size=48639) 2024-11-10T18:14:23,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-10T18:14:23,555 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262523575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262523575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262523575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:23,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=361, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1a82e239115744b3ae8f9e64f40a5042 2024-11-10T18:14:23,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/6327edb20bf34e4a94bb816fac4d3ccb is 50, key is test_row_0/B:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:23,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742045_1221 (size=12301) 2024-11-10T18:14:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:24,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42904 deadline: 1731262524084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42874 deadline: 1731262524084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:24,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42914 deadline: 1731262524084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:14:24,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/6327edb20bf34e4a94bb816fac4d3ccb 2024-11-10T18:14:24,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/8fbc47601ad6407fb9fce04c813329c6 is 50, key is test_row_0/C:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:24,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742046_1222 (size=12301) 2024-11-10T18:14:24,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/8fbc47601ad6407fb9fce04c813329c6 2024-11-10T18:14:24,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/1a82e239115744b3ae8f9e64f40a5042 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042 2024-11-10T18:14:24,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042, entries=250, sequenceid=361, filesize=47.5 K 2024-11-10T18:14:24,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/6327edb20bf34e4a94bb816fac4d3ccb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/6327edb20bf34e4a94bb816fac4d3ccb 2024-11-10T18:14:24,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/6327edb20bf34e4a94bb816fac4d3ccb, entries=150, sequenceid=361, filesize=12.0 K 2024-11-10T18:14:24,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/8fbc47601ad6407fb9fce04c813329c6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/8fbc47601ad6407fb9fce04c813329c6 2024-11-10T18:14:24,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/8fbc47601ad6407fb9fce04c813329c6, entries=150, sequenceid=361, filesize=12.0 K 2024-11-10T18:14:24,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a31b277868914d792297968bfe99e4fc in 1666ms, sequenceid=361, compaction requested=true 2024-11-10T18:14:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:24,574 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T18:14:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a31b277868914d792297968bfe99e4fc:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:24,574 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T18:14:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:24,576 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 174414 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T18:14:24,576 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T18:14:24,576 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/A is initiating minor compaction (all files) 2024-11-10T18:14:24,576 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/B is initiating minor compaction (all files) 2024-11-10T18:14:24,576 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/B in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:24,576 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/A in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:24,576 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/082eab1f43e4464db0ae33ecfc22862a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/21ef03eb30a64e799d61af2c0b26bf6b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e58cf688d4f242469db5d19a882a2f5e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e6af39e7b5894d4ea706f6aff59aad0b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/6327edb20bf34e4a94bb816fac4d3ccb] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=60.7 K 2024-11-10T18:14:24,576 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bcb631e460b8417da8a85c03071c4bea, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=170.3 K 2024-11-10T18:14:24,576 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:24,576 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bcb631e460b8417da8a85c03071c4bea, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042] 2024-11-10T18:14:24,576 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 082eab1f43e4464db0ae33ecfc22862a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731262454427 2024-11-10T18:14:24,577 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcb631e460b8417da8a85c03071c4bea, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731262454427 2024-11-10T18:14:24,577 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 21ef03eb30a64e799d61af2c0b26bf6b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731262456689 2024-11-10T18:14:24,577 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a75f73a6de641a7822e83cf7930f718, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731262456689 2024-11-10T18:14:24,577 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e58cf688d4f242469db5d19a882a2f5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731262457335 2024-11-10T18:14:24,577 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3aa9ba4e2dab4ebd85475a94bffa8f88, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731262457335 2024-11-10T18:14:24,577 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e6af39e7b5894d4ea706f6aff59aad0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731262459560 2024-11-10T18:14:24,578 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c01948581a84e77aac1d75305f66d8f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731262459560 2024-11-10T18:14:24,578 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6327edb20bf34e4a94bb816fac4d3ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731262461741 2024-11-10T18:14:24,578 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a82e239115744b3ae8f9e64f40a5042, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731262461739 2024-11-10T18:14:24,588 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:24,589 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#B#compaction#183 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:14:24,590 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110788eb20e3d4f4953b5524124438b69bb_a31b277868914d792297968bfe99e4fc store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:24,590 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/a742135290a24eeab3015bbb2b7efc45 is 50, key is test_row_0/B:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:24,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742047_1223 (size=13119) 2024-11-10T18:14:24,605 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110788eb20e3d4f4953b5524124438b69bb_a31b277868914d792297968bfe99e4fc, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:24,605 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110788eb20e3d4f4953b5524124438b69bb_a31b277868914d792297968bfe99e4fc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:24,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742048_1224 (size=4469) 2024-11-10T18:14:24,885 DEBUG [regionserver/691ea56905d2:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.15384615384615385, tune throughput to 57.69 MB/second 2024-11-10T18:14:24,999 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/a742135290a24eeab3015bbb2b7efc45 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/a742135290a24eeab3015bbb2b7efc45 2024-11-10T18:14:25,003 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a31b277868914d792297968bfe99e4fc/B of a31b277868914d792297968bfe99e4fc into a742135290a24eeab3015bbb2b7efc45(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:25,003 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:25,004 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/B, priority=11, startTime=1731262464574; duration=0sec 2024-11-10T18:14:25,004 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:25,004 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:B 2024-11-10T18:14:25,004 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-10T18:14:25,005 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62153 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-10T18:14:25,005 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): a31b277868914d792297968bfe99e4fc/C is initiating minor compaction (all files) 2024-11-10T18:14:25,005 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a31b277868914d792297968bfe99e4fc/C in TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:25,005 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/4d290dc025d247d8b0fde56861a015da, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bf65327a4cd94affab7e37cba4999d08, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7dad95c4462443a08fff55bf8ce39138, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7a5713f91c5343829cf5aab4109a7457, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/8fbc47601ad6407fb9fce04c813329c6] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp, totalSize=60.7 K 2024-11-10T18:14:25,006 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d290dc025d247d8b0fde56861a015da, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731262454427 2024-11-10T18:14:25,006 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting bf65327a4cd94affab7e37cba4999d08, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731262456689 2024-11-10T18:14:25,006 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7dad95c4462443a08fff55bf8ce39138, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731262457335 2024-11-10T18:14:25,007 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a5713f91c5343829cf5aab4109a7457, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731262459560 2024-11-10T18:14:25,007 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fbc47601ad6407fb9fce04c813329c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731262461741 2024-11-10T18:14:25,011 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#A#compaction#184 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:25,011 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/167b91e5f8eb433eb837517e5c54c2df is 175, key is test_row_0/A:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:25,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742049_1225 (size=32073) 2024-11-10T18:14:25,017 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a31b277868914d792297968bfe99e4fc#C#compaction#185 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:25,018 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/5032a95e6e68484fbc9bcde90041681c is 50, key is test_row_0/C:col10/1731262461741/Put/seqid=0 2024-11-10T18:14:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742050_1226 (size=13119) 2024-11-10T18:14:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:25,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-10T18:14:25,097 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-10T18:14:25,097 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:25,097 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:25,097 INFO [Thread-642 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-642. 2024-11-10T18:14:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:25,097 DEBUG [Thread-642 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:25,097 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:25,097 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:25,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:25,097 DEBUG [Thread-642 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:25,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:25,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:25,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110927667cfb3ab434ca880295764a9ade7_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262462936/Put/seqid=0 2024-11-10T18:14:25,106 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-10T18:14:25,106 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:25,106 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:25,106 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:25,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:25,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742051_1227 (size=12454) 2024-11-10T18:14:25,420 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/167b91e5f8eb433eb837517e5c54c2df as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/167b91e5f8eb433eb837517e5c54c2df 2024-11-10T18:14:25,425 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/5032a95e6e68484fbc9bcde90041681c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/5032a95e6e68484fbc9bcde90041681c 2024-11-10T18:14:25,425 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a31b277868914d792297968bfe99e4fc/A of a31b277868914d792297968bfe99e4fc into 167b91e5f8eb433eb837517e5c54c2df(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:25,425 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:25,426 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/A, priority=11, startTime=1731262464574; duration=0sec 2024-11-10T18:14:25,426 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:25,426 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:A 2024-11-10T18:14:25,429 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a31b277868914d792297968bfe99e4fc/C of a31b277868914d792297968bfe99e4fc into 5032a95e6e68484fbc9bcde90041681c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:25,429 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:25,429 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., storeName=a31b277868914d792297968bfe99e4fc/C, priority=11, startTime=1731262464574; duration=0sec 2024-11-10T18:14:25,430 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:25,430 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a31b277868914d792297968bfe99e4fc:C 2024-11-10T18:14:25,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:25,514 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110927667cfb3ab434ca880295764a9ade7_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110927667cfb3ab434ca880295764a9ade7_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:25,515 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/2066f4a304174b72b10b7ca550048b60, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:25,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/2066f4a304174b72b10b7ca550048b60 is 175, key is test_row_0/A:col10/1731262462936/Put/seqid=0 2024-11-10T18:14:25,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742052_1228 (size=31255) 2024-11-10T18:14:25,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/2066f4a304174b72b10b7ca550048b60 2024-11-10T18:14:25,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3eb2bb6d16f24d29b4c78a025fcdbed3 is 50, key is test_row_0/B:col10/1731262462936/Put/seqid=0 2024-11-10T18:14:25,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742053_1229 (size=12301) 2024-11-10T18:14:26,331 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3eb2bb6d16f24d29b4c78a025fcdbed3 2024-11-10T18:14:26,338 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f30e8ee7207f4d1692b7f3608d9889b1 is 50, key is test_row_0/C:col10/1731262462936/Put/seqid=0 2024-11-10T18:14:26,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742054_1230 (size=12301) 2024-11-10T18:14:26,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f30e8ee7207f4d1692b7f3608d9889b1 2024-11-10T18:14:26,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/2066f4a304174b72b10b7ca550048b60 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/2066f4a304174b72b10b7ca550048b60 2024-11-10T18:14:26,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/2066f4a304174b72b10b7ca550048b60, entries=150, sequenceid=384, filesize=30.5 K 2024-11-10T18:14:26,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/3eb2bb6d16f24d29b4c78a025fcdbed3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3eb2bb6d16f24d29b4c78a025fcdbed3 2024-11-10T18:14:26,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3eb2bb6d16f24d29b4c78a025fcdbed3, entries=150, sequenceid=384, filesize=12.0 K 2024-11-10T18:14:26,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/f30e8ee7207f4d1692b7f3608d9889b1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f30e8ee7207f4d1692b7f3608d9889b1 2024-11-10T18:14:26,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f30e8ee7207f4d1692b7f3608d9889b1, entries=150, sequenceid=384, filesize=12.0 K 2024-11-10T18:14:26,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=6.71 KB/6870 for a31b277868914d792297968bfe99e4fc in 1663ms, sequenceid=384, compaction requested=false 2024-11-10T18:14:26,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a31b277868914d792297968bfe99e4fc: 2024-11-10T18:14:29,886 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-10T18:14:29,886 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:29,886 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:29,886 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:29,886 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:29,896 INFO [Thread-640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-640. 2024-11-10T18:14:29,896 DEBUG [Thread-640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:29,896 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:29,896 DEBUG [Thread-640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:29,896 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 69 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2938 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2756 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1169 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3507 rows 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1173 2024-11-10T18:14:29,897 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3515 rows 2024-11-10T18:14:29,897 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:14:29,897 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:14:29,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:29,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:29,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:29,898 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T18:14:29,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T18:14:29,901 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262469901"}]},"ts":"1731262469901"} 2024-11-10T18:14:29,902 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T18:14:29,902 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T18:14:29,903 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:14:29,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, UNASSIGN}] 2024-11-10T18:14:29,905 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, UNASSIGN 2024-11-10T18:14:29,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:29,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, UNASSIGN because future has completed 2024-11-10T18:14:29,907 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:14:29,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:14:30,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T18:14:30,059 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:30,059 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:14:30,059 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing a31b277868914d792297968bfe99e4fc, disabling compactions & flushes 2024-11-10T18:14:30,059 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:30,059 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. after waiting 0 ms 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:30,060 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing a31b277868914d792297968bfe99e4fc 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=A 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=B 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a31b277868914d792297968bfe99e4fc, store=C 2024-11-10T18:14:30,060 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:30,065 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bb9d2124da9a48c89ad8f65ef9edf3f1_a31b277868914d792297968bfe99e4fc is 50, key is test_row_0/A:col10/1731262469885/Put/seqid=0 2024-11-10T18:14:30,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742055_1231 (size=12454) 2024-11-10T18:14:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T18:14:30,470 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:30,474 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110bb9d2124da9a48c89ad8f65ef9edf3f1_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bb9d2124da9a48c89ad8f65ef9edf3f1_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:30,475 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/6bdb1be1dd2e4a9aa4670eafcc214312, store: [table=TestAcidGuarantees family=A region=a31b277868914d792297968bfe99e4fc] 2024-11-10T18:14:30,475 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/6bdb1be1dd2e4a9aa4670eafcc214312 is 175, key is test_row_0/A:col10/1731262469885/Put/seqid=0 2024-11-10T18:14:30,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742056_1232 (size=31255) 2024-11-10T18:14:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T18:14:30,879 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/6bdb1be1dd2e4a9aa4670eafcc214312 2024-11-10T18:14:30,886 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/29bc7ca48ff748f787bc5baf02c24204 is 50, key is test_row_0/B:col10/1731262469885/Put/seqid=0 2024-11-10T18:14:30,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742057_1233 (size=12301) 2024-11-10T18:14:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T18:14:31,290 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/29bc7ca48ff748f787bc5baf02c24204 2024-11-10T18:14:31,296 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bb38129bd4544e45887f1fbc85465366 is 50, key is test_row_0/C:col10/1731262469885/Put/seqid=0 2024-11-10T18:14:31,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742058_1234 (size=12301) 2024-11-10T18:14:31,700 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bb38129bd4544e45887f1fbc85465366 2024-11-10T18:14:31,704 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/A/6bdb1be1dd2e4a9aa4670eafcc214312 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/6bdb1be1dd2e4a9aa4670eafcc214312 2024-11-10T18:14:31,707 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/6bdb1be1dd2e4a9aa4670eafcc214312, entries=150, sequenceid=392, filesize=30.5 K 2024-11-10T18:14:31,708 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/B/29bc7ca48ff748f787bc5baf02c24204 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/29bc7ca48ff748f787bc5baf02c24204 2024-11-10T18:14:31,711 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/29bc7ca48ff748f787bc5baf02c24204, entries=150, sequenceid=392, filesize=12.0 K 2024-11-10T18:14:31,712 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/.tmp/C/bb38129bd4544e45887f1fbc85465366 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bb38129bd4544e45887f1fbc85465366 2024-11-10T18:14:31,715 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bb38129bd4544e45887f1fbc85465366, entries=150, sequenceid=392, filesize=12.0 K 2024-11-10T18:14:31,715 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a31b277868914d792297968bfe99e4fc in 1655ms, sequenceid=392, compaction requested=true 2024-11-10T18:14:31,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/aa8abbac578a46958f3c1f42f884acb0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/9797924d71cf4e3ebfe3e71d94c4de98, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3467415f2a446638c3c442a2dc67beb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bb5ac4985f074fc4a32b89f9ea08eb9c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/34c6cb79ae104479875a510c708eb496, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bcb631e460b8417da8a85c03071c4bea, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042] to archive 2024-11-10T18:14:31,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:14:31,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/c585a1111e6d4d4f898ef9f8b340f052 2024-11-10T18:14:31,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/21e24fc511774f7faffca8800d631b3c 2024-11-10T18:14:31,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d556255a5c924e0f96b54fc350685c98 2024-11-10T18:14:31,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/aa8abbac578a46958f3c1f42f884acb0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/aa8abbac578a46958f3c1f42f884acb0 2024-11-10T18:14:31,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/8be0bc7f297a4d47b8b4a6f7ebb3e92e 2024-11-10T18:14:31,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/5718aefb5b974b0484ec50abec9e6ac4 2024-11-10T18:14:31,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/9797924d71cf4e3ebfe3e71d94c4de98 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/9797924d71cf4e3ebfe3e71d94c4de98 2024-11-10T18:14:31,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3a0fdb20ca94c038dd751ede0201fc8 2024-11-10T18:14:31,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0dbcd6a6dd294247b3b6415206f3e846 2024-11-10T18:14:31,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3467415f2a446638c3c442a2dc67beb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/d3467415f2a446638c3c442a2dc67beb 2024-11-10T18:14:31,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/141ab9406b4e49ee9ca68a69e3d0d936 2024-11-10T18:14:31,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bf49b007c2dd474e99bef950d038a48d 2024-11-10T18:14:31,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/694571a79ded49f78a1073dff544ca01 2024-11-10T18:14:31,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bb5ac4985f074fc4a32b89f9ea08eb9c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bb5ac4985f074fc4a32b89f9ea08eb9c 2024-11-10T18:14:31,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/26830ac412a74bba993419deaaa570dc 2024-11-10T18:14:31,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0b8c4b41833e4792a4cdfafbc94aea91 2024-11-10T18:14:31,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/34c6cb79ae104479875a510c708eb496 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/34c6cb79ae104479875a510c708eb496 2024-11-10T18:14:31,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/b75e395163f24e05b40fe6360cd16d24 2024-11-10T18:14:31,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/352c84dff5d44062bc008293c1adf5c9 2024-11-10T18:14:31,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bcb631e460b8417da8a85c03071c4bea to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/bcb631e460b8417da8a85c03071c4bea 2024-11-10T18:14:31,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/0a75f73a6de641a7822e83cf7930f718 2024-11-10T18:14:31,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/3aa9ba4e2dab4ebd85475a94bffa8f88 2024-11-10T18:14:31,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1c01948581a84e77aac1d75305f66d8f 2024-11-10T18:14:31,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/1a82e239115744b3ae8f9e64f40a5042 2024-11-10T18:14:31,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/0bbed838443a4f07ac48dfb76490b8a2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/95adc40d2cc34084af3b2481dbc22bb1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d38200c0a6f34347855094d252c7271e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/94355c6695cc4ba9a5c7ed9217542dbb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/93126532b8c544eaab6f7281268d8428, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d87a1acdae0945a3ae23b9ee2124347d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/cb2ab6b4d2a24761845cf7b4d4f5a50d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d9cc61137a6648e18619673ddd0c62f4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/455c0dbf9331468e928e04e0ed931afa, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d2813db46ff4b129b1e17de02914207, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b41a11638e794326ab478a974edf455b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/c71894cc6bb644ccbfe9c3b29665f5a2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e1ccbde3512e49189dc626e4458fcbc0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/09edf518650f4c50b8e75d2fd53f6d33, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d4414daf678466a80df0d1d4fe99b92, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d6e405051c6f4e26800c3a5576df61b1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/597501df797a41d8b52cda88c7b47fe3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b353ae1d77fe48da8370bf5036962f0d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/082eab1f43e4464db0ae33ecfc22862a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/9b6eaef1b6b94e058012c9fee45322cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/21ef03eb30a64e799d61af2c0b26bf6b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e58cf688d4f242469db5d19a882a2f5e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e6af39e7b5894d4ea706f6aff59aad0b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/6327edb20bf34e4a94bb816fac4d3ccb] to archive 2024-11-10T18:14:31,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:14:31,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/0bbed838443a4f07ac48dfb76490b8a2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/0bbed838443a4f07ac48dfb76490b8a2 2024-11-10T18:14:31,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/95adc40d2cc34084af3b2481dbc22bb1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/95adc40d2cc34084af3b2481dbc22bb1 2024-11-10T18:14:31,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d38200c0a6f34347855094d252c7271e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d38200c0a6f34347855094d252c7271e 2024-11-10T18:14:31,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/94355c6695cc4ba9a5c7ed9217542dbb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/94355c6695cc4ba9a5c7ed9217542dbb 2024-11-10T18:14:31,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/93126532b8c544eaab6f7281268d8428 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/93126532b8c544eaab6f7281268d8428 2024-11-10T18:14:31,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d87a1acdae0945a3ae23b9ee2124347d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d87a1acdae0945a3ae23b9ee2124347d 2024-11-10T18:14:31,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/cb2ab6b4d2a24761845cf7b4d4f5a50d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/cb2ab6b4d2a24761845cf7b4d4f5a50d 2024-11-10T18:14:31,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d9cc61137a6648e18619673ddd0c62f4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d9cc61137a6648e18619673ddd0c62f4 2024-11-10T18:14:31,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/455c0dbf9331468e928e04e0ed931afa to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/455c0dbf9331468e928e04e0ed931afa 2024-11-10T18:14:31,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d2813db46ff4b129b1e17de02914207 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d2813db46ff4b129b1e17de02914207 2024-11-10T18:14:31,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b41a11638e794326ab478a974edf455b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b41a11638e794326ab478a974edf455b 2024-11-10T18:14:31,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/c71894cc6bb644ccbfe9c3b29665f5a2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/c71894cc6bb644ccbfe9c3b29665f5a2 2024-11-10T18:14:31,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e1ccbde3512e49189dc626e4458fcbc0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e1ccbde3512e49189dc626e4458fcbc0 2024-11-10T18:14:31,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/09edf518650f4c50b8e75d2fd53f6d33 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/09edf518650f4c50b8e75d2fd53f6d33 2024-11-10T18:14:31,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d4414daf678466a80df0d1d4fe99b92 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3d4414daf678466a80df0d1d4fe99b92 2024-11-10T18:14:31,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d6e405051c6f4e26800c3a5576df61b1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/d6e405051c6f4e26800c3a5576df61b1 2024-11-10T18:14:31,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/597501df797a41d8b52cda88c7b47fe3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/597501df797a41d8b52cda88c7b47fe3 2024-11-10T18:14:31,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b353ae1d77fe48da8370bf5036962f0d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/b353ae1d77fe48da8370bf5036962f0d 2024-11-10T18:14:31,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/082eab1f43e4464db0ae33ecfc22862a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/082eab1f43e4464db0ae33ecfc22862a 2024-11-10T18:14:31,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/9b6eaef1b6b94e058012c9fee45322cd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/9b6eaef1b6b94e058012c9fee45322cd 2024-11-10T18:14:31,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/21ef03eb30a64e799d61af2c0b26bf6b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/21ef03eb30a64e799d61af2c0b26bf6b 2024-11-10T18:14:31,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e58cf688d4f242469db5d19a882a2f5e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e58cf688d4f242469db5d19a882a2f5e 2024-11-10T18:14:31,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e6af39e7b5894d4ea706f6aff59aad0b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/e6af39e7b5894d4ea706f6aff59aad0b 2024-11-10T18:14:31,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/6327edb20bf34e4a94bb816fac4d3ccb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/6327edb20bf34e4a94bb816fac4d3ccb 2024-11-10T18:14:31,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/abb33989671a44d1aa36353021f0a3d5, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/e6c023cc261744b281b2457f2af1c216, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/cf0c63152cf249b58d1b116503a37a1d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c0c9fa812f1643d6a7bbb863bc64a3cc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d86faa4983842b589ae3103834e2b8e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/a168481359094db299f8d1edd8abc073, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/ec4f88b2dfa34b55b6a43784edc65fe9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/d5542250559d45eb90338f159a3aad58, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bc4bbba8cafa4046a56cade6960165d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/93185440af0d4b5abd8a5559a7dcc456, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/1a765d5a23f9450e8093a44c17a7cb3f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f55bb38e8e3f431db87ab4d0bb2f7ff9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d5e1f107257468b90b119c49384b857, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/36aed4e35e0946c1b8d583c3b9396f94, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/b8a500b1ae2c4200870c9d4eaae2a7e6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/47b347e4a20346ada99dfba6ab221c20, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f4d8fe86cbae484c860151467f98c388, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/124fbbf766a743578ee6bde38db6c4d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/4d290dc025d247d8b0fde56861a015da, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c177ac0d4aa64849adb26894232dfd16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bf65327a4cd94affab7e37cba4999d08, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7dad95c4462443a08fff55bf8ce39138, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7a5713f91c5343829cf5aab4109a7457, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/8fbc47601ad6407fb9fce04c813329c6] to archive 2024-11-10T18:14:31,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:14:31,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/abb33989671a44d1aa36353021f0a3d5 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/abb33989671a44d1aa36353021f0a3d5 2024-11-10T18:14:31,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/e6c023cc261744b281b2457f2af1c216 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/e6c023cc261744b281b2457f2af1c216 2024-11-10T18:14:31,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/cf0c63152cf249b58d1b116503a37a1d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/cf0c63152cf249b58d1b116503a37a1d 2024-11-10T18:14:31,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c0c9fa812f1643d6a7bbb863bc64a3cc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c0c9fa812f1643d6a7bbb863bc64a3cc 2024-11-10T18:14:31,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d86faa4983842b589ae3103834e2b8e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d86faa4983842b589ae3103834e2b8e 2024-11-10T18:14:31,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/a168481359094db299f8d1edd8abc073 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/a168481359094db299f8d1edd8abc073 2024-11-10T18:14:31,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/ec4f88b2dfa34b55b6a43784edc65fe9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/ec4f88b2dfa34b55b6a43784edc65fe9 2024-11-10T18:14:31,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/d5542250559d45eb90338f159a3aad58 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/d5542250559d45eb90338f159a3aad58 2024-11-10T18:14:31,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bc4bbba8cafa4046a56cade6960165d3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bc4bbba8cafa4046a56cade6960165d3 2024-11-10T18:14:31,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/93185440af0d4b5abd8a5559a7dcc456 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/93185440af0d4b5abd8a5559a7dcc456 2024-11-10T18:14:31,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/1a765d5a23f9450e8093a44c17a7cb3f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/1a765d5a23f9450e8093a44c17a7cb3f 2024-11-10T18:14:31,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f55bb38e8e3f431db87ab4d0bb2f7ff9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f55bb38e8e3f431db87ab4d0bb2f7ff9 2024-11-10T18:14:31,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d5e1f107257468b90b119c49384b857 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/3d5e1f107257468b90b119c49384b857 2024-11-10T18:14:31,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/36aed4e35e0946c1b8d583c3b9396f94 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/36aed4e35e0946c1b8d583c3b9396f94 2024-11-10T18:14:31,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/b8a500b1ae2c4200870c9d4eaae2a7e6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/b8a500b1ae2c4200870c9d4eaae2a7e6 2024-11-10T18:14:31,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/47b347e4a20346ada99dfba6ab221c20 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/47b347e4a20346ada99dfba6ab221c20 2024-11-10T18:14:31,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f4d8fe86cbae484c860151467f98c388 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f4d8fe86cbae484c860151467f98c388 2024-11-10T18:14:31,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/124fbbf766a743578ee6bde38db6c4d3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/124fbbf766a743578ee6bde38db6c4d3 2024-11-10T18:14:31,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/4d290dc025d247d8b0fde56861a015da to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/4d290dc025d247d8b0fde56861a015da 2024-11-10T18:14:31,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c177ac0d4aa64849adb26894232dfd16 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/c177ac0d4aa64849adb26894232dfd16 2024-11-10T18:14:31,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bf65327a4cd94affab7e37cba4999d08 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bf65327a4cd94affab7e37cba4999d08 2024-11-10T18:14:31,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7dad95c4462443a08fff55bf8ce39138 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7dad95c4462443a08fff55bf8ce39138 2024-11-10T18:14:31,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7a5713f91c5343829cf5aab4109a7457 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/7a5713f91c5343829cf5aab4109a7457 2024-11-10T18:14:31,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/8fbc47601ad6407fb9fce04c813329c6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/8fbc47601ad6407fb9fce04c813329c6 2024-11-10T18:14:31,798 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/recovered.edits/395.seqid, newMaxSeqId=395, maxSeqId=4 2024-11-10T18:14:31,799 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. 2024-11-10T18:14:31,799 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for a31b277868914d792297968bfe99e4fc: Waiting for close lock at 1731262470059Running coprocessor pre-close hooks at 1731262470059Disabling compacts and flushes for region at 1731262470059Disabling writes for close at 1731262470060 (+1 ms)Obtaining lock to block concurrent updates at 1731262470060Preparing flush snapshotting stores in a31b277868914d792297968bfe99e4fc at 1731262470060Finished memstore snapshotting TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731262470060Flushing stores of TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc. at 1731262470061 (+1 ms)Flushing a31b277868914d792297968bfe99e4fc/A: creating writer at 1731262470061Flushing mob file a31b277868914d792297968bfe99e4fc/A: appending metadata at 1731262470065 (+4 ms)Flushing mob file a31b277868914d792297968bfe99e4fc/A: closing flushed file at 1731262470065Flushing a31b277868914d792297968bfe99e4fc/A: appending metadata at 1731262470475 (+410 ms)Flushing a31b277868914d792297968bfe99e4fc/A: closing flushed file at 1731262470475Flushing a31b277868914d792297968bfe99e4fc/B: creating writer at 1731262470883 (+408 ms)Flushing a31b277868914d792297968bfe99e4fc/B: appending metadata at 1731262470885 (+2 ms)Flushing a31b277868914d792297968bfe99e4fc/B: closing flushed file at 1731262470885Flushing a31b277868914d792297968bfe99e4fc/C: creating writer at 1731262471293 (+408 ms)Flushing a31b277868914d792297968bfe99e4fc/C: appending metadata at 1731262471295 (+2 ms)Flushing a31b277868914d792297968bfe99e4fc/C: closing flushed file at 1731262471295Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31c0f630: reopening flushed file at 1731262471703 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18cb0cbc: reopening flushed file at 1731262471707 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28cde4c: reopening flushed file at 1731262471711 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for a31b277868914d792297968bfe99e4fc in 1655ms, sequenceid=392, compaction requested=true at 1731262471715 (+4 ms)Writing region close event to WAL at 1731262471796 (+81 ms)Running coprocessor post-close hooks at 1731262471799 (+3 ms)Closed at 1731262471799 2024-11-10T18:14:31,801 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:31,802 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=a31b277868914d792297968bfe99e4fc, regionState=CLOSED 2024-11-10T18:14:31,803 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:14:31,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-10T18:14:31,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure a31b277868914d792297968bfe99e4fc, server=691ea56905d2,32909,1731262402364 in 1.8970 sec 2024-11-10T18:14:31,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-10T18:14:31,807 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a31b277868914d792297968bfe99e4fc, UNASSIGN in 1.9010 sec 2024-11-10T18:14:31,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-10T18:14:31,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9040 sec 2024-11-10T18:14:31,810 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262471810"}]},"ts":"1731262471810"} 2024-11-10T18:14:31,811 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T18:14:31,811 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T18:14:31,813 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9140 sec 2024-11-10T18:14:32,018 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T18:14:32,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-10T18:14:32,045 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:32,046 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T18:14:32,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,047 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T18:14:32,048 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,051 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,053 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/recovered.edits] 2024-11-10T18:14:32,056 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/167b91e5f8eb433eb837517e5c54c2df to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/167b91e5f8eb433eb837517e5c54c2df 2024-11-10T18:14:32,057 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/2066f4a304174b72b10b7ca550048b60 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/2066f4a304174b72b10b7ca550048b60 2024-11-10T18:14:32,058 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/6bdb1be1dd2e4a9aa4670eafcc214312 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/A/6bdb1be1dd2e4a9aa4670eafcc214312 2024-11-10T18:14:32,060 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/29bc7ca48ff748f787bc5baf02c24204 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/29bc7ca48ff748f787bc5baf02c24204 2024-11-10T18:14:32,061 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3eb2bb6d16f24d29b4c78a025fcdbed3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/3eb2bb6d16f24d29b4c78a025fcdbed3 2024-11-10T18:14:32,062 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/a742135290a24eeab3015bbb2b7efc45 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/B/a742135290a24eeab3015bbb2b7efc45 2024-11-10T18:14:32,064 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/5032a95e6e68484fbc9bcde90041681c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/5032a95e6e68484fbc9bcde90041681c 2024-11-10T18:14:32,065 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bb38129bd4544e45887f1fbc85465366 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/bb38129bd4544e45887f1fbc85465366 2024-11-10T18:14:32,066 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f30e8ee7207f4d1692b7f3608d9889b1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/C/f30e8ee7207f4d1692b7f3608d9889b1 2024-11-10T18:14:32,069 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/recovered.edits/395.seqid to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc/recovered.edits/395.seqid 2024-11-10T18:14:32,069 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,069 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T18:14:32,070 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T18:14:32,070 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T18:14:32,074 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101c8b08b95df04ace8fdae7f9bb439377_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101c8b08b95df04ace8fdae7f9bb439377_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,075 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101d73b46f86724a148564803568b15513_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101d73b46f86724a148564803568b15513_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,076 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101da97df86d0f4f2c82b3cc0627dd84d4_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411101da97df86d0f4f2c82b3cc0627dd84d4_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,077 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102050fd62bed4405c8ce773940b8ae2e0_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102050fd62bed4405c8ce773940b8ae2e0_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,078 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111022edc569759a44e48e7a4eb041713c01_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111022edc569759a44e48e7a4eb041713c01_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,078 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c0c6b8c45f9464795410996c0106f23_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c0c6b8c45f9464795410996c0106f23_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,079 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107caa5409b75c482d8aa56f6f3a36827f_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107caa5409b75c482d8aa56f6f3a36827f_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,080 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d4c3957dbd441189a61deab666f4193_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107d4c3957dbd441189a61deab666f4193_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,082 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111082fa4446e558432cbb421996ff5d45f8_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111082fa4446e558432cbb421996ff5d45f8_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,083 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110900f64e764b647558195355b7ea4fc8a_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110900f64e764b647558195355b7ea4fc8a_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,084 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110927667cfb3ab434ca880295764a9ade7_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110927667cfb3ab434ca880295764a9ade7_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,085 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a03349e755944e2bb86e275d8b887e4_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411109a03349e755944e2bb86e275d8b887e4_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,086 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bb9d2124da9a48c89ad8f65ef9edf3f1_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bb9d2124da9a48c89ad8f65ef9edf3f1_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,087 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bbeab4a7a5604bd4a9dd5f310764b844_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bbeab4a7a5604bd4a9dd5f310764b844_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,088 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be40c3a317e740498712b522333f279e_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be40c3a317e740498712b522333f279e_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,089 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be74515d7b0749cfa8faf38a2e6d3fe2_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110be74515d7b0749cfa8faf38a2e6d3fe2_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bf58244a7ff04f2c809707d28153e964_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110bf58244a7ff04f2c809707d28153e964_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,090 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e6eecdc5c3e44de68bdf7a1f07049227_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e6eecdc5c3e44de68bdf7a1f07049227_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,091 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f3ff6546b2ed467dbbed570b196193bf_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110f3ff6546b2ed467dbbed570b196193bf_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,092 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff19984adbde4a47aa2f8805c068b25c_a31b277868914d792297968bfe99e4fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ff19984adbde4a47aa2f8805c068b25c_a31b277868914d792297968bfe99e4fc 2024-11-10T18:14:32,093 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T18:14:32,095 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,098 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T18:14:32,100 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T18:14:32,101 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,101 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T18:14:32,101 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731262472101"}]},"ts":"9223372036854775807"} 2024-11-10T18:14:32,103 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T18:14:32,103 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a31b277868914d792297968bfe99e4fc, NAME => 'TestAcidGuarantees,,1731262440027.a31b277868914d792297968bfe99e4fc.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T18:14:32,103 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T18:14:32,103 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731262472103"}]},"ts":"9223372036854775807"} 2024-11-10T18:14:32,105 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T18:14:32,105 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-10T18:14:32,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-10T18:14:32,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T18:14:32,155 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:32,164 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1086771773_22 at /127.0.0.1:55754 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=333 (was 343), ProcessCount=11 (was 11), AvailableMemoryMB=8109 (was 8207) 2024-11-10T18:14:32,173 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=333, ProcessCount=11, AvailableMemoryMB=8109 2024-11-10T18:14:32,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:14:32,174 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:14:32,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:32,176 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T18:14:32,176 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:32,176 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-10T18:14:32,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T18:14:32,177 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T18:14:32,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742059_1235 (size=1061) 2024-11-10T18:14:32,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T18:14:32,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T18:14:32,584 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:14:32,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742060_1236 (size=53) 2024-11-10T18:14:32,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T18:14:32,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:14:32,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8db0c68c9c5ead4c454abb5a20de2443, disabling compactions & flushes 2024-11-10T18:14:32,990 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:32,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:32,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. after waiting 0 ms 2024-11-10T18:14:32,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:32,990 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:32,990 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8db0c68c9c5ead4c454abb5a20de2443: Waiting for close lock at 1731262472990Disabling compacts and flushes for region at 1731262472990Disabling writes for close at 1731262472990Writing region close event to WAL at 1731262472990Closed at 1731262472990 2024-11-10T18:14:32,992 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T18:14:32,992 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731262472992"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731262472992"}]},"ts":"1731262472992"} 2024-11-10T18:14:32,993 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T18:14:32,994 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T18:14:32,994 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262472994"}]},"ts":"1731262472994"} 2024-11-10T18:14:32,996 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T18:14:32,996 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, ASSIGN}] 2024-11-10T18:14:32,997 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, ASSIGN 2024-11-10T18:14:32,997 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, ASSIGN; state=OFFLINE, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=false 2024-11-10T18:14:33,148 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8db0c68c9c5ead4c454abb5a20de2443, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:33,150 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, ASSIGN because future has completed 2024-11-10T18:14:33,150 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:14:33,305 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:33,305 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:14:33,305 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,305 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:14:33,305 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,305 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,306 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,307 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:33,308 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8db0c68c9c5ead4c454abb5a20de2443 columnFamilyName A 2024-11-10T18:14:33,308 DEBUG [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:33,308 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.HStore(327): Store=8db0c68c9c5ead4c454abb5a20de2443/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:33,308 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,309 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:33,309 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8db0c68c9c5ead4c454abb5a20de2443 columnFamilyName B 2024-11-10T18:14:33,310 DEBUG [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:33,310 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.HStore(327): Store=8db0c68c9c5ead4c454abb5a20de2443/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:33,310 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,311 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:14:33,311 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8db0c68c9c5ead4c454abb5a20de2443 columnFamilyName C 2024-11-10T18:14:33,311 DEBUG [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:14:33,311 INFO [StoreOpener-8db0c68c9c5ead4c454abb5a20de2443-1 {}] regionserver.HStore(327): Store=8db0c68c9c5ead4c454abb5a20de2443/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:14:33,312 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:33,312 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,312 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,313 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,313 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,313 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,314 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:14:33,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T18:14:33,315 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,317 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:14:33,318 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 8db0c68c9c5ead4c454abb5a20de2443; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74238345, jitterRate=0.10623754560947418}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:14:33,318 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:33,318 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 8db0c68c9c5ead4c454abb5a20de2443: Running coprocessor pre-open hook at 1731262473305Writing region info on filesystem at 1731262473305Initializing all the Stores at 1731262473306 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262473306Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262473306Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262473306Cleaning up temporary data from old regions at 1731262473313 (+7 ms)Running coprocessor post-open hooks at 1731262473318 (+5 ms)Region opened successfully at 1731262473318 2024-11-10T18:14:33,319 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., pid=63, masterSystemTime=1731262473302 2024-11-10T18:14:33,321 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:33,321 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:33,322 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=8db0c68c9c5ead4c454abb5a20de2443, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:33,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:14:33,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-10T18:14:33,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 in 173 msec 2024-11-10T18:14:33,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-10T18:14:33,327 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, ASSIGN in 329 msec 2024-11-10T18:14:33,327 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T18:14:33,328 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262473327"}]},"ts":"1731262473327"} 2024-11-10T18:14:33,329 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T18:14:33,329 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T18:14:33,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-10T18:14:34,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-10T18:14:34,325 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:34,326 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,326 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,327 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0a6dc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,328 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,329 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,329 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2758775d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,330 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,332 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:14:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:14:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3e108666 2024-11-10T18:14:34,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:14:34,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:14:34,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,335 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,335 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1db434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,337 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f881e2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,339 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54519fd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593066f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,341 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,341 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,341 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,342 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33090d8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39469d2e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15f31982, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f24a056, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,348 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302d49ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c5f78df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,351 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3144a911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,352 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57092773, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,353 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,353 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,353 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136019f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,355 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53fe5486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,356 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,356 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653e0eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,358 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f68f839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,358 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,358 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,358 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49fabb22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf56444, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:14:34,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:14:34,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b358cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:14:34,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:34,363 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57488, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:14:34,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@404e7e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:14:34,364 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,364 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,364 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,364 DEBUG [Thread-1107 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,364 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,364 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:34,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,366 DEBUG [Thread-1105 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,366 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,366 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-10T18:14:34,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42468, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42476, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,367 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T18:14:34,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42484, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,368 DEBUG [Thread-1118 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,368 DEBUG [Thread-1116 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:14:34,368 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42508, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:14:34,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:14:34,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:14:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,371 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,374 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:14:34,376 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:34,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:34,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:34,380 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:34,380 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:34,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/7882aacec6604d4cba13412c1b32b8c6 is 50, key is test_row_1/A:col10/1731262474377/Put/seqid=0 2024-11-10T18:14:34,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742061_1237 (size=9657) 2024-11-10T18:14:34,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262474426, max timestamp=1731262474435Num uniques 859; 2024-11-10T18:14:34,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262534431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:14:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731262474426, max timestamp=1731262474435Num uniques 859; 2024-11-10T18:14:34,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T18:14:34,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731262474426, max timestamp=1731262474435Num uniques 859; 2024-11-10T18:14:34,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T18:14:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T18:14:34,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262534469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262534473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262534473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262534476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,481 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T18:14:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262534544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262534585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262534585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262534585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262534585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T18:14:34,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T18:14:34,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262534755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262534794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262534794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262534794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262534794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:34,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:34,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/7882aacec6604d4cba13412c1b32b8c6 2024-11-10T18:14:34,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d1d4c2d28df4404bbbf3da7e7d16a88f is 50, key is test_row_1/B:col10/1731262474377/Put/seqid=0 2024-11-10T18:14:34,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T18:14:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:34,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742062_1238 (size=9657) 2024-11-10T18:14:34,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d1d4c2d28df4404bbbf3da7e7d16a88f 2024-11-10T18:14:34,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7243269aab3c478a9746b42b09fe6f46 is 50, key is test_row_1/C:col10/1731262474377/Put/seqid=0 2024-11-10T18:14:34,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742063_1239 (size=9657) 2024-11-10T18:14:34,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7243269aab3c478a9746b42b09fe6f46 2024-11-10T18:14:34,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/7882aacec6604d4cba13412c1b32b8c6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/7882aacec6604d4cba13412c1b32b8c6 2024-11-10T18:14:34,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/7882aacec6604d4cba13412c1b32b8c6, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T18:14:34,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d1d4c2d28df4404bbbf3da7e7d16a88f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d1d4c2d28df4404bbbf3da7e7d16a88f 2024-11-10T18:14:34,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d1d4c2d28df4404bbbf3da7e7d16a88f, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T18:14:34,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7243269aab3c478a9746b42b09fe6f46 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7243269aab3c478a9746b42b09fe6f46 2024-11-10T18:14:34,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7243269aab3c478a9746b42b09fe6f46, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T18:14:34,956 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.63 KB/138880 for 8db0c68c9c5ead4c454abb5a20de2443 in 577ms, sequenceid=11, compaction requested=false 2024-11-10T18:14:34,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T18:14:34,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-10T18:14:34,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:34,996 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=135.63 KB heapSize=388.23 KB 2024-11-10T18:14:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:34,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/350c7a90e1de4df4ab7801a67f52a0af is 50, key is test_row_0/A:col10/1731262474467/Put/seqid=0 2024-11-10T18:14:35,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742064_1240 (size=12001) 2024-11-10T18:14:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:35,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:35,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262535098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262535104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262535104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262535104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262535104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,111 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262535204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,415 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/350c7a90e1de4df4ab7801a67f52a0af 2024-11-10T18:14:35,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262535415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/5bd4f82e59ae4b7a82b69d3840a32064 is 50, key is test_row_0/B:col10/1731262474467/Put/seqid=0 2024-11-10T18:14:35,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742065_1241 (size=12001) 2024-11-10T18:14:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T18:14:35,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262535615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262535615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,618 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262535615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262535624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262535725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:35,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:35,833 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/5bd4f82e59ae4b7a82b69d3840a32064 2024-11-10T18:14:35,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/b91d37c093d246f7b19eb21360577d69 is 50, key is test_row_0/C:col10/1731262474467/Put/seqid=0 2024-11-10T18:14:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742066_1242 (size=12001) 2024-11-10T18:14:36,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262536234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,247 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/b91d37c093d246f7b19eb21360577d69 2024-11-10T18:14:36,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/350c7a90e1de4df4ab7801a67f52a0af as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/350c7a90e1de4df4ab7801a67f52a0af 2024-11-10T18:14:36,257 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/350c7a90e1de4df4ab7801a67f52a0af, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T18:14:36,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/5bd4f82e59ae4b7a82b69d3840a32064 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/5bd4f82e59ae4b7a82b69d3840a32064 2024-11-10T18:14:36,262 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/5bd4f82e59ae4b7a82b69d3840a32064, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T18:14:36,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/b91d37c093d246f7b19eb21360577d69 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/b91d37c093d246f7b19eb21360577d69 2024-11-10T18:14:36,267 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/b91d37c093d246f7b19eb21360577d69, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T18:14:36,268 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~135.63 KB/138880, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for 8db0c68c9c5ead4c454abb5a20de2443 in 1272ms, sequenceid=48, compaction requested=false 2024-11-10T18:14:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-10T18:14:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-10T18:14:36,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-10T18:14:36,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8900 sec 2024-11-10T18:14:36,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.9080 sec 2024-11-10T18:14:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-10T18:14:36,515 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:36,515 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-10T18:14:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T18:14:36,522 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:36,523 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:36,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T18:14:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:36,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:14:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:36,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:36,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/86a810544bcd42a39609e0ee25daea5a is 50, key is test_row_0/A:col10/1731262476626/Put/seqid=0 2024-11-10T18:14:36,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742067_1243 (size=14341) 2024-11-10T18:14:36,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:36,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262536679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,684 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262536683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262536683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262536684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262536794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262536794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262536795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262536795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:36,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:36,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:36,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T18:14:36,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:36,983 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:36,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262537005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262537005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262537005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262537005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/86a810544bcd42a39609e0ee25daea5a 2024-11-10T18:14:37,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/52e96629da30416c883fd0e720bb0c6e is 50, key is test_row_0/B:col10/1731262476626/Put/seqid=0 2024-11-10T18:14:37,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742068_1244 (size=12001) 2024-11-10T18:14:37,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/52e96629da30416c883fd0e720bb0c6e 2024-11-10T18:14:37,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/03d7ff0b3ba949b99902649cbc9761b7 is 50, key is test_row_0/C:col10/1731262476626/Put/seqid=0 2024-11-10T18:14:37,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742069_1245 (size=12001) 2024-11-10T18:14:37,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:37,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T18:14:37,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262537255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,259 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262537314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262537315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262537315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262537315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,323 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:14:37,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:37,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/03d7ff0b3ba949b99902649cbc9761b7 2024-11-10T18:14:37,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/86a810544bcd42a39609e0ee25daea5a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/86a810544bcd42a39609e0ee25daea5a 2024-11-10T18:14:37,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/86a810544bcd42a39609e0ee25daea5a, entries=200, sequenceid=60, filesize=14.0 K 2024-11-10T18:14:37,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/52e96629da30416c883fd0e720bb0c6e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e96629da30416c883fd0e720bb0c6e 2024-11-10T18:14:37,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e96629da30416c883fd0e720bb0c6e, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T18:14:37,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/03d7ff0b3ba949b99902649cbc9761b7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/03d7ff0b3ba949b99902649cbc9761b7 2024-11-10T18:14:37,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/03d7ff0b3ba949b99902649cbc9761b7, entries=150, sequenceid=60, filesize=11.7 K 2024-11-10T18:14:37,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8db0c68c9c5ead4c454abb5a20de2443 in 860ms, sequenceid=60, compaction requested=true 2024-11-10T18:14:37,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:37,487 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:37,487 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:37,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:37,489 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:37,489 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:37,489 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:37,489 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:37,489 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,489 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,489 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d1d4c2d28df4404bbbf3da7e7d16a88f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/5bd4f82e59ae4b7a82b69d3840a32064, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e96629da30416c883fd0e720bb0c6e] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=32.9 K 2024-11-10T18:14:37,489 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/7882aacec6604d4cba13412c1b32b8c6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/350c7a90e1de4df4ab7801a67f52a0af, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/86a810544bcd42a39609e0ee25daea5a] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.2 K 2024-11-10T18:14:37,490 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d4c2d28df4404bbbf3da7e7d16a88f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731262474377 2024-11-10T18:14:37,490 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7882aacec6604d4cba13412c1b32b8c6, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731262474377 2024-11-10T18:14:37,490 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 350c7a90e1de4df4ab7801a67f52a0af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731262474467 2024-11-10T18:14:37,490 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd4f82e59ae4b7a82b69d3840a32064, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731262474467 2024-11-10T18:14:37,491 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86a810544bcd42a39609e0ee25daea5a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262475091 2024-11-10T18:14:37,491 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 52e96629da30416c883fd0e720bb0c6e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262475091 2024-11-10T18:14:37,502 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#201 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:37,503 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/f010a1b8f57e4959900879a2ba5d95de is 50, key is test_row_0/A:col10/1731262476626/Put/seqid=0 2024-11-10T18:14:37,504 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:37,505 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/09152455a07d4e278ae94ae17eb2ab91 is 50, key is test_row_0/B:col10/1731262476626/Put/seqid=0 2024-11-10T18:14:37,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742071_1247 (size=12104) 2024-11-10T18:14:37,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742070_1246 (size=12104) 2024-11-10T18:14:37,529 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/09152455a07d4e278ae94ae17eb2ab91 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/09152455a07d4e278ae94ae17eb2ab91 2024-11-10T18:14:37,533 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/f010a1b8f57e4959900879a2ba5d95de as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f010a1b8f57e4959900879a2ba5d95de 2024-11-10T18:14:37,536 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into 09152455a07d4e278ae94ae17eb2ab91(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:37,536 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:37,536 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262477487; duration=0sec 2024-11-10T18:14:37,536 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:37,536 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:37,536 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:37,538 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:37,538 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:37,539 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,539 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7243269aab3c478a9746b42b09fe6f46, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/b91d37c093d246f7b19eb21360577d69, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/03d7ff0b3ba949b99902649cbc9761b7] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=32.9 K 2024-11-10T18:14:37,540 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7243269aab3c478a9746b42b09fe6f46, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731262474377 2024-11-10T18:14:37,540 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into f010a1b8f57e4959900879a2ba5d95de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:37,540 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:37,540 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262477486; duration=0sec 2024-11-10T18:14:37,540 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:37,540 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:37,540 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b91d37c093d246f7b19eb21360577d69, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731262474467 2024-11-10T18:14:37,540 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d7ff0b3ba949b99902649cbc9761b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262475091 2024-11-10T18:14:37,553 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:37,554 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/63d3d249d18647cf87ddd51fdcc8ed18 is 50, key is test_row_0/C:col10/1731262476626/Put/seqid=0 2024-11-10T18:14:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742072_1248 (size=12104) 2024-11-10T18:14:37,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-10T18:14:37,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:37,597 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:14:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:37,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/60c14841c1544bd6bebe0efbc9cec92a is 50, key is test_row_0/A:col10/1731262476681/Put/seqid=0 2024-11-10T18:14:37,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742073_1249 (size=12001) 2024-11-10T18:14:37,611 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/60c14841c1544bd6bebe0efbc9cec92a 2024-11-10T18:14:37,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/285f65bdd83f41f0beab966bffc6b13a is 50, key is test_row_0/B:col10/1731262476681/Put/seqid=0 2024-11-10T18:14:37,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742074_1250 (size=12001) 2024-11-10T18:14:37,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T18:14:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:37,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:37,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262537835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262537836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,840 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,841 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262537839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262537840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262537945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,947 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262537945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262537945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262537954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:37,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:37,989 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/63d3d249d18647cf87ddd51fdcc8ed18 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/63d3d249d18647cf87ddd51fdcc8ed18 2024-11-10T18:14:37,994 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into 63d3d249d18647cf87ddd51fdcc8ed18(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:37,994 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:37,994 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262477487; duration=0sec 2024-11-10T18:14:37,994 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:37,994 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:38,030 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/285f65bdd83f41f0beab966bffc6b13a 2024-11-10T18:14:38,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/625c70f518944a47ad342fd59a25a1e6 is 50, key is test_row_0/C:col10/1731262476681/Put/seqid=0 2024-11-10T18:14:38,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742075_1251 (size=12001) 2024-11-10T18:14:38,044 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/625c70f518944a47ad342fd59a25a1e6 2024-11-10T18:14:38,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/60c14841c1544bd6bebe0efbc9cec92a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/60c14841c1544bd6bebe0efbc9cec92a 2024-11-10T18:14:38,052 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/60c14841c1544bd6bebe0efbc9cec92a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T18:14:38,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/285f65bdd83f41f0beab966bffc6b13a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/285f65bdd83f41f0beab966bffc6b13a 2024-11-10T18:14:38,057 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/285f65bdd83f41f0beab966bffc6b13a, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T18:14:38,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/625c70f518944a47ad342fd59a25a1e6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/625c70f518944a47ad342fd59a25a1e6 2024-11-10T18:14:38,062 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/625c70f518944a47ad342fd59a25a1e6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-10T18:14:38,063 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8db0c68c9c5ead4c454abb5a20de2443 in 467ms, sequenceid=82, compaction requested=false 2024-11-10T18:14:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:38,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-10T18:14:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-10T18:14:38,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-10T18:14:38,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5410 sec 2024-11-10T18:14:38,069 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.5520 sec 2024-11-10T18:14:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:38,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:14:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:38,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:38,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/b74f717f19b24a1e9fcb34139677cf7f is 50, key is test_row_0/A:col10/1731262477837/Put/seqid=0 2024-11-10T18:14:38,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742076_1252 (size=12001) 2024-11-10T18:14:38,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262538178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262538178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262538180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262538184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262538294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262538295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262538295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,298 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262538295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262538504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262538505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262538505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262538505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,509 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/b74f717f19b24a1e9fcb34139677cf7f 2024-11-10T18:14:38,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/0144623e54fa4bee934d65f2f67902f9 is 50, key is test_row_0/B:col10/1731262477837/Put/seqid=0 2024-11-10T18:14:38,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742077_1253 (size=12001) 2024-11-10T18:14:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-10T18:14:38,664 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:38,665 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:38,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-10T18:14:38,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T18:14:38,667 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:38,668 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:38,668 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:38,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T18:14:38,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262538814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262538815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262538815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262538815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:38,820 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:38,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T18:14:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:38,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:38,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:38,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T18:14:38,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:38,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:38,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:38,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:38,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:38,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:38,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/0144623e54fa4bee934d65f2f67902f9 2024-11-10T18:14:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T18:14:39,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/829f285a22c947a29410d665713c74cd is 50, key is test_row_0/C:col10/1731262477837/Put/seqid=0 2024-11-10T18:14:39,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742078_1254 (size=12001) 2024-11-10T18:14:39,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/829f285a22c947a29410d665713c74cd 2024-11-10T18:14:39,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/b74f717f19b24a1e9fcb34139677cf7f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/b74f717f19b24a1e9fcb34139677cf7f 2024-11-10T18:14:39,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/b74f717f19b24a1e9fcb34139677cf7f, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T18:14:39,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/0144623e54fa4bee934d65f2f67902f9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/0144623e54fa4bee934d65f2f67902f9 2024-11-10T18:14:39,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/0144623e54fa4bee934d65f2f67902f9, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T18:14:39,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/829f285a22c947a29410d665713c74cd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/829f285a22c947a29410d665713c74cd 2024-11-10T18:14:39,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/829f285a22c947a29410d665713c74cd, entries=150, sequenceid=96, filesize=11.7 K 2024-11-10T18:14:39,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8db0c68c9c5ead4c454abb5a20de2443 in 876ms, sequenceid=96, compaction requested=true 2024-11-10T18:14:39,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:39,033 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:39,033 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:39,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:39,034 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:39,034 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:39,034 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,034 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:39,034 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:39,034 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f010a1b8f57e4959900879a2ba5d95de, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/60c14841c1544bd6bebe0efbc9cec92a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/b74f717f19b24a1e9fcb34139677cf7f] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.3 K 2024-11-10T18:14:39,034 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,034 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/09152455a07d4e278ae94ae17eb2ab91, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/285f65bdd83f41f0beab966bffc6b13a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/0144623e54fa4bee934d65f2f67902f9] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.3 K 2024-11-10T18:14:39,035 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 09152455a07d4e278ae94ae17eb2ab91, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262475091 2024-11-10T18:14:39,035 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f010a1b8f57e4959900879a2ba5d95de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262475091 2024-11-10T18:14:39,035 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 285f65bdd83f41f0beab966bffc6b13a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731262476681 2024-11-10T18:14:39,035 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60c14841c1544bd6bebe0efbc9cec92a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731262476681 2024-11-10T18:14:39,036 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0144623e54fa4bee934d65f2f67902f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731262477837 2024-11-10T18:14:39,036 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b74f717f19b24a1e9fcb34139677cf7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731262477837 2024-11-10T18:14:39,045 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:39,046 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e2885791266c4fb9864090881c419cf9 is 50, key is test_row_0/B:col10/1731262477837/Put/seqid=0 2024-11-10T18:14:39,049 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:39,049 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c1919d3fedd14294b434e32ed2127469 is 50, key is test_row_0/A:col10/1731262477837/Put/seqid=0 2024-11-10T18:14:39,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742079_1255 (size=12207) 2024-11-10T18:14:39,068 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e2885791266c4fb9864090881c419cf9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e2885791266c4fb9864090881c419cf9 2024-11-10T18:14:39,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742080_1256 (size=12207) 2024-11-10T18:14:39,075 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into e2885791266c4fb9864090881c419cf9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:39,075 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:39,075 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262479033; duration=0sec 2024-11-10T18:14:39,075 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:39,075 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:39,076 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:39,080 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:39,080 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:39,080 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,080 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/63d3d249d18647cf87ddd51fdcc8ed18, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/625c70f518944a47ad342fd59a25a1e6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/829f285a22c947a29410d665713c74cd] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.3 K 2024-11-10T18:14:39,081 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 63d3d249d18647cf87ddd51fdcc8ed18, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731262475091 2024-11-10T18:14:39,082 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 625c70f518944a47ad342fd59a25a1e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731262476681 2024-11-10T18:14:39,084 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 829f285a22c947a29410d665713c74cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731262477837 2024-11-10T18:14:39,084 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c1919d3fedd14294b434e32ed2127469 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c1919d3fedd14294b434e32ed2127469 2024-11-10T18:14:39,092 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into c1919d3fedd14294b434e32ed2127469(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:39,092 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:39,092 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262479032; duration=0sec 2024-11-10T18:14:39,092 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:39,092 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:39,097 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#212 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:39,097 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/c6d071d8e91541d1aee54e8f65fe79fd is 50, key is test_row_0/C:col10/1731262477837/Put/seqid=0 2024-11-10T18:14:39,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742081_1257 (size=12207) 2024-11-10T18:14:39,115 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/c6d071d8e91541d1aee54e8f65fe79fd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/c6d071d8e91541d1aee54e8f65fe79fd 2024-11-10T18:14:39,121 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into c6d071d8e91541d1aee54e8f65fe79fd(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:39,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:39,122 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262479033; duration=0sec 2024-11-10T18:14:39,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:39,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:39,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-10T18:14:39,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,127 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T18:14:39,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:39,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:39,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/df4f1c652e954268988165e8d5ffa50d is 50, key is test_row_0/A:col10/1731262478182/Put/seqid=0 2024-11-10T18:14:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742082_1258 (size=12001) 2024-11-10T18:14:39,152 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/df4f1c652e954268988165e8d5ffa50d 2024-11-10T18:14:39,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/35ed70e4b92e407cb32a5f2f4a39b05d is 50, key is test_row_0/B:col10/1731262478182/Put/seqid=0 2024-11-10T18:14:39,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742083_1259 (size=12001) 2024-11-10T18:14:39,187 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/35ed70e4b92e407cb32a5f2f4a39b05d 2024-11-10T18:14:39,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/34de94a6306e4d0b9c0104b791616399 is 50, key is test_row_0/C:col10/1731262478182/Put/seqid=0 2024-11-10T18:14:39,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742084_1260 (size=12001) 2024-11-10T18:14:39,216 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/34de94a6306e4d0b9c0104b791616399 2024-11-10T18:14:39,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/df4f1c652e954268988165e8d5ffa50d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/df4f1c652e954268988165e8d5ffa50d 2024-11-10T18:14:39,229 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/df4f1c652e954268988165e8d5ffa50d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T18:14:39,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/35ed70e4b92e407cb32a5f2f4a39b05d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/35ed70e4b92e407cb32a5f2f4a39b05d 2024-11-10T18:14:39,235 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/35ed70e4b92e407cb32a5f2f4a39b05d, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T18:14:39,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/34de94a6306e4d0b9c0104b791616399 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/34de94a6306e4d0b9c0104b791616399 2024-11-10T18:14:39,242 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/34de94a6306e4d0b9c0104b791616399, entries=150, sequenceid=118, filesize=11.7 K 2024-11-10T18:14:39,243 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 8db0c68c9c5ead4c454abb5a20de2443 in 116ms, sequenceid=118, compaction requested=false 2024-11-10T18:14:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-10T18:14:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-10T18:14:39,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-10T18:14:39,247 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 577 msec 2024-11-10T18:14:39,250 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 583 msec 2024-11-10T18:14:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:39,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:39,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-10T18:14:39,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/bc0a059c396e40febea2f8f00c310ed0 is 50, key is test_row_0/A:col10/1731262479286/Put/seqid=0 2024-11-10T18:14:39,305 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:39,306 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-10T18:14:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T18:14:39,310 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:39,311 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:39,311 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:39,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742085_1261 (size=12001) 2024-11-10T18:14:39,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/bc0a059c396e40febea2f8f00c310ed0 2024-11-10T18:14:39,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/eb7c32715b3946dab343935778e12867 is 50, key is test_row_0/B:col10/1731262479286/Put/seqid=0 2024-11-10T18:14:39,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742086_1262 (size=12001) 2024-11-10T18:14:39,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262539342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262479330, max timestamp=1731262479348Num uniques 859; 2024-11-10T18:14:39,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,352 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262539344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262479330, max timestamp=1731262479348Num uniques 859; 2024-11-10T18:14:39,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731262479330, max timestamp=1731262479348Num uniques 859; 2024-11-10T18:14:39,357 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-10T18:14:39,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262539388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,395 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,398 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731262479362, max timestamp=1731262479393Num uniques 1018; 2024-11-10T18:14:39,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T18:14:39,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731262479362, max timestamp=1731262479393Num uniques 1009; 2024-11-10T18:14:39,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-10T18:14:39,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731262479362, max timestamp=1731262479393Num uniques 1018; 2024-11-10T18:14:39,399 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-10T18:14:39,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T18:14:39,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T18:14:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262539461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262539464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,468 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262539468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262539468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262539504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,509 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262539574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262539584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262539585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T18:14:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:39,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T18:14:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262539674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262539715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/eb7c32715b3946dab343935778e12867 2024-11-10T18:14:39,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/688249fe0d594d72a3a98d22a5c8d1ce is 50, key is test_row_0/C:col10/1731262479286/Put/seqid=0 2024-11-10T18:14:39,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742087_1263 (size=12001) 2024-11-10T18:14:39,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T18:14:39,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262539784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262539795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262539795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:39,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T18:14:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:39,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T18:14:39,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262539984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:39,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:39,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262540025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T18:14:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:40,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:40,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:40,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:40,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262540095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262540105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262540105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,109 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/688249fe0d594d72a3a98d22a5c8d1ce 2024-11-10T18:14:40,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/bc0a059c396e40febea2f8f00c310ed0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/bc0a059c396e40febea2f8f00c310ed0 2024-11-10T18:14:40,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/bc0a059c396e40febea2f8f00c310ed0, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T18:14:40,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/eb7c32715b3946dab343935778e12867 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/eb7c32715b3946dab343935778e12867 2024-11-10T18:14:40,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/eb7c32715b3946dab343935778e12867, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T18:14:40,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/688249fe0d594d72a3a98d22a5c8d1ce as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/688249fe0d594d72a3a98d22a5c8d1ce 2024-11-10T18:14:40,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/688249fe0d594d72a3a98d22a5c8d1ce, entries=150, sequenceid=128, filesize=11.7 K 2024-11-10T18:14:40,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 8db0c68c9c5ead4c454abb5a20de2443 in 873ms, sequenceid=128, compaction requested=true 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:40,169 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:40,169 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:40,169 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:40,170 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:40,170 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:40,170 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:40,170 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c1919d3fedd14294b434e32ed2127469, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/df4f1c652e954268988165e8d5ffa50d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/bc0a059c396e40febea2f8f00c310ed0] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.4 K 2024-11-10T18:14:40,170 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:40,171 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:40,171 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:40,171 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e2885791266c4fb9864090881c419cf9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/35ed70e4b92e407cb32a5f2f4a39b05d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/eb7c32715b3946dab343935778e12867] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.4 K 2024-11-10T18:14:40,171 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e2885791266c4fb9864090881c419cf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731262477837 2024-11-10T18:14:40,171 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1919d3fedd14294b434e32ed2127469, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731262477837 2024-11-10T18:14:40,172 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting df4f1c652e954268988165e8d5ffa50d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731262478174 2024-11-10T18:14:40,172 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ed70e4b92e407cb32a5f2f4a39b05d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731262478174 2024-11-10T18:14:40,172 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc0a059c396e40febea2f8f00c310ed0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731262479286 2024-11-10T18:14:40,172 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting eb7c32715b3946dab343935778e12867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731262479286 2024-11-10T18:14:40,186 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:40,186 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:40,187 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e09ebe3e27954b80878429eafb532a56 is 50, key is test_row_0/A:col10/1731262479286/Put/seqid=0 2024-11-10T18:14:40,187 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/8e814dcf09d146ed9de2a2501e4f4ed3 is 50, key is test_row_0/B:col10/1731262479286/Put/seqid=0 2024-11-10T18:14:40,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742089_1265 (size=12309) 2024-11-10T18:14:40,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742088_1264 (size=12309) 2024-11-10T18:14:40,205 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/8e814dcf09d146ed9de2a2501e4f4ed3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8e814dcf09d146ed9de2a2501e4f4ed3 2024-11-10T18:14:40,209 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e09ebe3e27954b80878429eafb532a56 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e09ebe3e27954b80878429eafb532a56 2024-11-10T18:14:40,211 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into 8e814dcf09d146ed9de2a2501e4f4ed3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:40,211 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:40,211 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262480169; duration=0sec 2024-11-10T18:14:40,212 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:40,212 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:40,212 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:40,214 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:40,214 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:40,214 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:40,214 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/c6d071d8e91541d1aee54e8f65fe79fd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/34de94a6306e4d0b9c0104b791616399, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/688249fe0d594d72a3a98d22a5c8d1ce] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.4 K 2024-11-10T18:14:40,215 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c6d071d8e91541d1aee54e8f65fe79fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731262477837 2024-11-10T18:14:40,216 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 34de94a6306e4d0b9c0104b791616399, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731262478174 2024-11-10T18:14:40,216 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 688249fe0d594d72a3a98d22a5c8d1ce, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731262479286 2024-11-10T18:14:40,216 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into e09ebe3e27954b80878429eafb532a56(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:40,216 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:40,217 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262480169; duration=0sec 2024-11-10T18:14:40,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:40,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:40,225 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:40,226 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ce456e60ff9945eea7a67b1aa50744a6 is 50, key is test_row_0/C:col10/1731262479286/Put/seqid=0 2024-11-10T18:14:40,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742090_1266 (size=12309) 2024-11-10T18:14:40,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-10T18:14:40,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:40,232 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=140.89 KB heapSize=399.38 KB 2024-11-10T18:14:40,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:40,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:40,235 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ce456e60ff9945eea7a67b1aa50744a6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ce456e60ff9945eea7a67b1aa50744a6 2024-11-10T18:14:40,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/07fe0187c78346debdcc51f96ce41d48 is 50, key is test_row_0/A:col10/1731262479466/Put/seqid=0 2024-11-10T18:14:40,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742091_1267 (size=12151) 2024-11-10T18:14:40,244 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/07fe0187c78346debdcc51f96ce41d48 2024-11-10T18:14:40,245 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into ce456e60ff9945eea7a67b1aa50744a6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:40,245 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:40,245 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262480169; duration=0sec 2024-11-10T18:14:40,245 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:40,245 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:40,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3a7c83e4e71c4fd6bcf9a875935c8990 is 50, key is test_row_0/B:col10/1731262479466/Put/seqid=0 2024-11-10T18:14:40,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742092_1268 (size=12151) 2024-11-10T18:14:40,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T18:14:40,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:40,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:40,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262540523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262540535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,538 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262540604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,608 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262540614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262540615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,619 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262540635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:40,660 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3a7c83e4e71c4fd6bcf9a875935c8990 2024-11-10T18:14:40,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/27a0d2e7ccb742de8c4dec32aa560c48 is 50, key is test_row_0/C:col10/1731262479466/Put/seqid=0 2024-11-10T18:14:40,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742093_1269 (size=12151) 2024-11-10T18:14:40,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:40,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262540845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:40,849 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,073 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/27a0d2e7ccb742de8c4dec32aa560c48 2024-11-10T18:14:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/07fe0187c78346debdcc51f96ce41d48 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/07fe0187c78346debdcc51f96ce41d48 2024-11-10T18:14:41,082 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/07fe0187c78346debdcc51f96ce41d48, entries=150, sequenceid=184, filesize=11.9 K 2024-11-10T18:14:41,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3a7c83e4e71c4fd6bcf9a875935c8990 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3a7c83e4e71c4fd6bcf9a875935c8990 2024-11-10T18:14:41,087 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3a7c83e4e71c4fd6bcf9a875935c8990, entries=150, sequenceid=184, filesize=11.9 K 2024-11-10T18:14:41,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/27a0d2e7ccb742de8c4dec32aa560c48 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/27a0d2e7ccb742de8c4dec32aa560c48 2024-11-10T18:14:41,092 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/27a0d2e7ccb742de8c4dec32aa560c48, entries=150, sequenceid=184, filesize=11.9 K 2024-11-10T18:14:41,093 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~399.33 KB/408912, currentSize=40.25 KB/41220 for 8db0c68c9c5ead4c454abb5a20de2443 in 861ms, sequenceid=184, compaction requested=false 2024-11-10T18:14:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-10T18:14:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-10T18:14:41,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-10T18:14:41,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7840 sec 2024-11-10T18:14:41,099 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7910 sec 2024-11-10T18:14:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:41,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:41,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:41,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:41,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:41,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:41,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:41,157 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:41,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e7662c8a94394c7ab98e3aa1723685b4 is 50, key is test_row_0/A:col10/1731262480501/Put/seqid=0 2024-11-10T18:14:41,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742094_1270 (size=12151) 2024-11-10T18:14:41,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262481226, max timestamp=1731262481246Num uniques 859; 2024-11-10T18:14:41,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262481226, max timestamp=1731262481246Num uniques 859; 2024-11-10T18:14:41,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:41,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:41,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262481226, max timestamp=1731262481246Num uniques 859; 2024-11-10T18:14:41,250 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:41,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262541343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-10T18:14:41,454 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:41,455 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-10T18:14:41,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T18:14:41,456 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:41,457 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:41,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:41,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262541454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262541545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,550 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T18:14:41,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e7662c8a94394c7ab98e3aa1723685b4 2024-11-10T18:14:41,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d6f6c43bb7e64a01ae72840c6e6fa795 is 50, key is test_row_0/B:col10/1731262480501/Put/seqid=0 2024-11-10T18:14:41,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742095_1271 (size=12151) 2024-11-10T18:14:41,591 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d6f6c43bb7e64a01ae72840c6e6fa795 2024-11-10T18:14:41,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/003e36f9b6884e24a7e3afd7a2f87943 is 50, key is test_row_0/C:col10/1731262480501/Put/seqid=0 2024-11-10T18:14:41,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742096_1272 (size=12151) 2024-11-10T18:14:41,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T18:14:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262541625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262541625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,629 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,629 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,629 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262541634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,638 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:41,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262541665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:41,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T18:14:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T18:14:41,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T18:14:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:41,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:41,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:41,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262541974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:41,977 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:42,003 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/003e36f9b6884e24a7e3afd7a2f87943 2024-11-10T18:14:42,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e7662c8a94394c7ab98e3aa1723685b4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e7662c8a94394c7ab98e3aa1723685b4 2024-11-10T18:14:42,012 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e7662c8a94394c7ab98e3aa1723685b4, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T18:14:42,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d6f6c43bb7e64a01ae72840c6e6fa795 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d6f6c43bb7e64a01ae72840c6e6fa795 2024-11-10T18:14:42,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d6f6c43bb7e64a01ae72840c6e6fa795, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T18:14:42,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/003e36f9b6884e24a7e3afd7a2f87943 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/003e36f9b6884e24a7e3afd7a2f87943 2024-11-10T18:14:42,018 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T18:14:42,018 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T18:14:42,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/003e36f9b6884e24a7e3afd7a2f87943, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T18:14:42,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8db0c68c9c5ead4c454abb5a20de2443 in 865ms, sequenceid=195, compaction requested=true 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:42,022 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:42,022 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:42,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:42,023 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:42,023 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:42,023 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:42,023 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8e814dcf09d146ed9de2a2501e4f4ed3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3a7c83e4e71c4fd6bcf9a875935c8990, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d6f6c43bb7e64a01ae72840c6e6fa795] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.8 K 2024-11-10T18:14:42,023 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:42,023 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:42,023 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:42,024 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e09ebe3e27954b80878429eafb532a56, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/07fe0187c78346debdcc51f96ce41d48, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e7662c8a94394c7ab98e3aa1723685b4] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.8 K 2024-11-10T18:14:42,024 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09ebe3e27954b80878429eafb532a56, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731262479286 2024-11-10T18:14:42,024 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e814dcf09d146ed9de2a2501e4f4ed3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731262479286 2024-11-10T18:14:42,024 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07fe0187c78346debdcc51f96ce41d48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731262479458 2024-11-10T18:14:42,024 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a7c83e4e71c4fd6bcf9a875935c8990, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731262479458 2024-11-10T18:14:42,025 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7662c8a94394c7ab98e3aa1723685b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262480501 2024-11-10T18:14:42,025 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f6c43bb7e64a01ae72840c6e6fa795, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262480501 2024-11-10T18:14:42,035 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:42,036 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/af85ebeee4f04bf6818ce3334cca0e2f is 50, key is test_row_0/B:col10/1731262480501/Put/seqid=0 2024-11-10T18:14:42,036 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:42,037 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6270f73d288a4121b60096119c5d1afc is 50, key is test_row_0/A:col10/1731262480501/Put/seqid=0 2024-11-10T18:14:42,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742097_1273 (size=12561) 2024-11-10T18:14:42,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742098_1274 (size=12561) 2024-11-10T18:14:42,051 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6270f73d288a4121b60096119c5d1afc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6270f73d288a4121b60096119c5d1afc 2024-11-10T18:14:42,056 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into 6270f73d288a4121b60096119c5d1afc(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:42,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:42,056 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262482022; duration=0sec 2024-11-10T18:14:42,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:42,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:42,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:42,058 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:42,058 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:42,058 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:42,058 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ce456e60ff9945eea7a67b1aa50744a6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/27a0d2e7ccb742de8c4dec32aa560c48, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/003e36f9b6884e24a7e3afd7a2f87943] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=35.8 K 2024-11-10T18:14:42,059 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce456e60ff9945eea7a67b1aa50744a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731262479286 2024-11-10T18:14:42,059 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 27a0d2e7ccb742de8c4dec32aa560c48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731262479458 2024-11-10T18:14:42,059 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 003e36f9b6884e24a7e3afd7a2f87943, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262480501 2024-11-10T18:14:42,067 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#230 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:42,067 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/1fa2277b42654c6e80f93b9eba3f12a7 is 50, key is test_row_0/C:col10/1731262480501/Put/seqid=0 2024-11-10T18:14:42,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-10T18:14:42,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:42,069 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T18:14:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:42,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:42,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742099_1275 (size=12561) 2024-11-10T18:14:42,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/83634311c86447b19183982583fd8667 is 50, key is test_row_0/A:col10/1731262481302/Put/seqid=0 2024-11-10T18:14:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T18:14:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742100_1276 (size=12151) 2024-11-10T18:14:42,086 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/83634311c86447b19183982583fd8667 2024-11-10T18:14:42,088 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/1fa2277b42654c6e80f93b9eba3f12a7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fa2277b42654c6e80f93b9eba3f12a7 2024-11-10T18:14:42,094 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into 1fa2277b42654c6e80f93b9eba3f12a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:42,095 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:42,095 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262482022; duration=0sec 2024-11-10T18:14:42,095 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:42,095 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/1552be142657461080a80652aea1ba76 is 50, key is test_row_0/B:col10/1731262481302/Put/seqid=0 2024-11-10T18:14:42,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742101_1277 (size=12151) 2024-11-10T18:14:42,448 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/af85ebeee4f04bf6818ce3334cca0e2f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/af85ebeee4f04bf6818ce3334cca0e2f 2024-11-10T18:14:42,453 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into af85ebeee4f04bf6818ce3334cca0e2f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:42,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:42,453 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262482022; duration=0sec 2024-11-10T18:14:42,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:42,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:42,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:42,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:42,503 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/1552be142657461080a80652aea1ba76 2024-11-10T18:14:42,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/84faa428621640d7baea28d0ad27e088 is 50, key is test_row_0/C:col10/1731262481302/Put/seqid=0 2024-11-10T18:14:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742102_1278 (size=12151) 2024-11-10T18:14:42,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:42,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262542519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:42,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:42,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:42,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T18:14:42,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:42,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262542634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:42,639 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:42,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:42,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262542844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:42,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:42,915 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/84faa428621640d7baea28d0ad27e088 2024-11-10T18:14:42,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/83634311c86447b19183982583fd8667 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/83634311c86447b19183982583fd8667 2024-11-10T18:14:42,922 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/83634311c86447b19183982583fd8667, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T18:14:42,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/1552be142657461080a80652aea1ba76 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/1552be142657461080a80652aea1ba76 2024-11-10T18:14:42,927 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/1552be142657461080a80652aea1ba76, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T18:14:42,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/84faa428621640d7baea28d0ad27e088 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/84faa428621640d7baea28d0ad27e088 2024-11-10T18:14:42,932 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/84faa428621640d7baea28d0ad27e088, entries=150, sequenceid=233, filesize=11.9 K 2024-11-10T18:14:42,933 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 8db0c68c9c5ead4c454abb5a20de2443 in 863ms, sequenceid=233, compaction requested=false 2024-11-10T18:14:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:42,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-10T18:14:42,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-10T18:14:42,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-10T18:14:42,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4770 sec 2024-11-10T18:14:42,938 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4820 sec 2024-11-10T18:14:43,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:43,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:14:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:43,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:43,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/fc9a3979dec14acdbf8c9557e5602efa is 50, key is test_row_0/A:col10/1731262482517/Put/seqid=0 2024-11-10T18:14:43,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742103_1279 (size=12151) 2024-11-10T18:14:43,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/fc9a3979dec14acdbf8c9557e5602efa 2024-11-10T18:14:43,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3bfd5ff749ee49b395a7820b3f51f384 is 50, key is test_row_0/B:col10/1731262482517/Put/seqid=0 2024-11-10T18:14:43,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742104_1280 (size=12151) 2024-11-10T18:14:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262483217, max timestamp=1731262483252Num uniques 859; 2024-11-10T18:14:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262483217, max timestamp=1731262483252Num uniques 859; 2024-11-10T18:14:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262483217, max timestamp=1731262483252Num uniques 859; 2024-11-10T18:14:43,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:43,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262543338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262543455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,459 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262543564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,568 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3bfd5ff749ee49b395a7820b3f51f384 2024-11-10T18:14:43,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/010092ccdabe4fab99dd682b02e44a4f is 50, key is test_row_0/C:col10/1731262482517/Put/seqid=0 2024-11-10T18:14:43,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742105_1281 (size=12151) 2024-11-10T18:14:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-10T18:14:43,605 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:43,606 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-10T18:14:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T18:14:43,607 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:43,608 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:43,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:43,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262543654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262543654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,660 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:43,660 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262543665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262543665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,667 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:43,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,668 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T18:14:43,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T18:14:43,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:43,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:43,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:43,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:43,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:43,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:43,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T18:14:43,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:43,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:43,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:43,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:43,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:43,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T18:14:43,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262543974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:43,979 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:43,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/010092ccdabe4fab99dd682b02e44a4f 2024-11-10T18:14:43,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/fc9a3979dec14acdbf8c9557e5602efa as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fc9a3979dec14acdbf8c9557e5602efa 2024-11-10T18:14:44,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fc9a3979dec14acdbf8c9557e5602efa, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T18:14:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3bfd5ff749ee49b395a7820b3f51f384 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3bfd5ff749ee49b395a7820b3f51f384 2024-11-10T18:14:44,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3bfd5ff749ee49b395a7820b3f51f384, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T18:14:44,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/010092ccdabe4fab99dd682b02e44a4f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/010092ccdabe4fab99dd682b02e44a4f 2024-11-10T18:14:44,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/010092ccdabe4fab99dd682b02e44a4f, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T18:14:44,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 8db0c68c9c5ead4c454abb5a20de2443 in 857ms, sequenceid=246, compaction requested=true 2024-11-10T18:14:44,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:44,015 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:44,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:44,015 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:44,016 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:44,016 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:44,016 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:44,016 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,016 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:44,016 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/af85ebeee4f04bf6818ce3334cca0e2f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/1552be142657461080a80652aea1ba76, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3bfd5ff749ee49b395a7820b3f51f384] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.0 K 2024-11-10T18:14:44,016 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,016 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6270f73d288a4121b60096119c5d1afc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/83634311c86447b19183982583fd8667, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fc9a3979dec14acdbf8c9557e5602efa] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.0 K 2024-11-10T18:14:44,016 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting af85ebeee4f04bf6818ce3334cca0e2f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262480501 2024-11-10T18:14:44,016 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6270f73d288a4121b60096119c5d1afc, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262480501 2024-11-10T18:14:44,017 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1552be142657461080a80652aea1ba76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731262481302 2024-11-10T18:14:44,017 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83634311c86447b19183982583fd8667, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731262481302 2024-11-10T18:14:44,017 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bfd5ff749ee49b395a7820b3f51f384, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262482497 2024-11-10T18:14:44,017 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc9a3979dec14acdbf8c9557e5602efa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262482497 2024-11-10T18:14:44,031 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:44,031 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:44,032 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/08d44e0c6b5040b0ae9f88e14061bd9e is 50, key is test_row_0/B:col10/1731262482517/Put/seqid=0 2024-11-10T18:14:44,032 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/d94c3fca66f54a03a1fd0fde4fea9db9 is 50, key is test_row_0/A:col10/1731262482517/Put/seqid=0 2024-11-10T18:14:44,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742107_1283 (size=12663) 2024-11-10T18:14:44,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742106_1282 (size=12663) 2024-11-10T18:14:44,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-10T18:14:44,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,067 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-10T18:14:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:44,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:14:44,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/a9744642d52043ca855a1d58a542b668 is 50, key is test_row_0/A:col10/1731262483331/Put/seqid=0 2024-11-10T18:14:44,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742108_1284 (size=12301) 2024-11-10T18:14:44,084 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/a9744642d52043ca855a1d58a542b668 2024-11-10T18:14:44,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/cf074c20798f46a384ec853e45a8d120 is 50, key is test_row_0/B:col10/1731262483331/Put/seqid=0 2024-11-10T18:14:44,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742109_1285 (size=12301) 2024-11-10T18:14:44,107 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/cf074c20798f46a384ec853e45a8d120 2024-11-10T18:14:44,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/5ba279c40ba34c42914d1c48abc7b206 is 50, key is test_row_0/C:col10/1731262483331/Put/seqid=0 2024-11-10T18:14:44,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742110_1286 (size=12301) 2024-11-10T18:14:44,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T18:14:44,449 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/d94c3fca66f54a03a1fd0fde4fea9db9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/d94c3fca66f54a03a1fd0fde4fea9db9 2024-11-10T18:14:44,455 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into d94c3fca66f54a03a1fd0fde4fea9db9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:44,455 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:44,455 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262484014; duration=0sec 2024-11-10T18:14:44,455 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:44,455 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:44,455 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:44,457 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/08d44e0c6b5040b0ae9f88e14061bd9e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/08d44e0c6b5040b0ae9f88e14061bd9e 2024-11-10T18:14:44,457 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:44,457 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:44,457 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,457 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fa2277b42654c6e80f93b9eba3f12a7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/84faa428621640d7baea28d0ad27e088, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/010092ccdabe4fab99dd682b02e44a4f] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.0 K 2024-11-10T18:14:44,458 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fa2277b42654c6e80f93b9eba3f12a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262480501 2024-11-10T18:14:44,459 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84faa428621640d7baea28d0ad27e088, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731262481302 2024-11-10T18:14:44,459 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 010092ccdabe4fab99dd682b02e44a4f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262482497 2024-11-10T18:14:44,463 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into 08d44e0c6b5040b0ae9f88e14061bd9e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:44,463 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:44,463 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262484015; duration=0sec 2024-11-10T18:14:44,463 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:44,463 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:44,466 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:44,467 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2a3afbaa90414170a72fcdc0f825ab24 is 50, key is test_row_0/C:col10/1731262482517/Put/seqid=0 2024-11-10T18:14:44,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742111_1287 (size=12663) 2024-11-10T18:14:44,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:44,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:44,523 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/5ba279c40ba34c42914d1c48abc7b206 2024-11-10T18:14:44,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/a9744642d52043ca855a1d58a542b668 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a9744642d52043ca855a1d58a542b668 2024-11-10T18:14:44,532 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a9744642d52043ca855a1d58a542b668, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T18:14:44,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/cf074c20798f46a384ec853e45a8d120 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/cf074c20798f46a384ec853e45a8d120 2024-11-10T18:14:44,537 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/cf074c20798f46a384ec853e45a8d120, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T18:14:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/5ba279c40ba34c42914d1c48abc7b206 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/5ba279c40ba34c42914d1c48abc7b206 2024-11-10T18:14:44,541 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/5ba279c40ba34c42914d1c48abc7b206, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T18:14:44,542 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 8db0c68c9c5ead4c454abb5a20de2443 in 475ms, sequenceid=282, compaction requested=false 2024-11-10T18:14:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:44,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-10T18:14:44,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-10T18:14:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:44,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:14:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:44,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:44,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-10T18:14:44,546 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 936 msec 2024-11-10T18:14:44,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 941 msec 2024-11-10T18:14:44,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/4779a34bcbeb4299b7a1cc46063ff7fc is 50, key is test_row_0/A:col10/1731262484518/Put/seqid=0 2024-11-10T18:14:44,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742112_1288 (size=14741) 2024-11-10T18:14:44,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:44,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262544637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:44,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:44,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-10T18:14:44,745 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:44,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262544745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:44,746 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:44,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-10T18:14:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:44,747 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:44,748 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:44,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:44,875 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2a3afbaa90414170a72fcdc0f825ab24 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2a3afbaa90414170a72fcdc0f825ab24 2024-11-10T18:14:44,880 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into 2a3afbaa90414170a72fcdc0f825ab24(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:44,880 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:44,880 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262484015; duration=0sec 2024-11-10T18:14:44,880 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:44,880 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:44,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:44,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:44,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:44,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:44,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:44,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/4779a34bcbeb4299b7a1cc46063ff7fc 2024-11-10T18:14:44,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:44,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262544955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:44,959 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:44,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3566e826513645c490129e86ce09d359 is 50, key is test_row_0/B:col10/1731262484518/Put/seqid=0 2024-11-10T18:14:44,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742113_1289 (size=12301) 2024-11-10T18:14:45,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:45,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:45,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:45,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:45,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262545264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:45,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3566e826513645c490129e86ce09d359 2024-11-10T18:14:45,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ae85fdc32b774842b6765d9bcbfa05be is 50, key is test_row_0/C:col10/1731262484518/Put/seqid=0 2024-11-10T18:14:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:45,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742114_1290 (size=12301) 2024-11-10T18:14:45,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:45,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:45,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:45,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:45,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262545774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:45,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:45,784 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ae85fdc32b774842b6765d9bcbfa05be 2024-11-10T18:14:45,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/4779a34bcbeb4299b7a1cc46063ff7fc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/4779a34bcbeb4299b7a1cc46063ff7fc 2024-11-10T18:14:45,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/4779a34bcbeb4299b7a1cc46063ff7fc, entries=200, sequenceid=296, filesize=14.4 K 2024-11-10T18:14:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3566e826513645c490129e86ce09d359 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3566e826513645c490129e86ce09d359 2024-11-10T18:14:45,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3566e826513645c490129e86ce09d359, entries=150, sequenceid=296, filesize=12.0 K 2024-11-10T18:14:45,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ae85fdc32b774842b6765d9bcbfa05be as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ae85fdc32b774842b6765d9bcbfa05be 2024-11-10T18:14:45,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ae85fdc32b774842b6765d9bcbfa05be, entries=150, sequenceid=296, filesize=12.0 K 2024-11-10T18:14:45,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 8db0c68c9c5ead4c454abb5a20de2443 in 1257ms, sequenceid=296, compaction requested=true 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:45,802 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:45,802 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:45,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:45,803 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39705 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:45,803 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:45,803 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,803 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:45,803 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/d94c3fca66f54a03a1fd0fde4fea9db9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a9744642d52043ca855a1d58a542b668, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/4779a34bcbeb4299b7a1cc46063ff7fc] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=38.8 K 2024-11-10T18:14:45,803 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:45,803 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,804 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/08d44e0c6b5040b0ae9f88e14061bd9e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/cf074c20798f46a384ec853e45a8d120, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3566e826513645c490129e86ce09d359] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.4 K 2024-11-10T18:14:45,804 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d94c3fca66f54a03a1fd0fde4fea9db9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262482497 2024-11-10T18:14:45,804 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 08d44e0c6b5040b0ae9f88e14061bd9e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262482497 2024-11-10T18:14:45,804 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9744642d52043ca855a1d58a542b668, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262483319 2024-11-10T18:14:45,804 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cf074c20798f46a384ec853e45a8d120, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262483319 2024-11-10T18:14:45,805 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4779a34bcbeb4299b7a1cc46063ff7fc, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731262484518 2024-11-10T18:14:45,805 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3566e826513645c490129e86ce09d359, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731262484518 2024-11-10T18:14:45,812 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:45,813 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/40d3ddc6e33f4aba888c5faf5da340c3 is 50, key is test_row_0/A:col10/1731262484518/Put/seqid=0 2024-11-10T18:14:45,814 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:45,815 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3288babfb6734559afd922515bd01f26 is 50, key is test_row_0/B:col10/1731262484518/Put/seqid=0 2024-11-10T18:14:45,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:45,820 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:45,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:45,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742116_1292 (size=12915) 2024-11-10T18:14:45,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742115_1291 (size=12915) 2024-11-10T18:14:45,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/2b9a5252e50544828d3ad409d005c5b7 is 50, key is test_row_0/A:col10/1731262484594/Put/seqid=0 2024-11-10T18:14:45,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742117_1293 (size=12301) 2024-11-10T18:14:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:46,231 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/40d3ddc6e33f4aba888c5faf5da340c3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/40d3ddc6e33f4aba888c5faf5da340c3 2024-11-10T18:14:46,231 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3288babfb6734559afd922515bd01f26 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3288babfb6734559afd922515bd01f26 2024-11-10T18:14:46,234 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/2b9a5252e50544828d3ad409d005c5b7 2024-11-10T18:14:46,236 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into 40d3ddc6e33f4aba888c5faf5da340c3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:46,236 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:46,236 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262485802; duration=0sec 2024-11-10T18:14:46,237 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:46,237 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:46,237 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into 3288babfb6734559afd922515bd01f26(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:46,237 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:46,237 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:46,237 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262485802; duration=0sec 2024-11-10T18:14:46,237 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:46,237 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:46,238 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:46,238 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:46,238 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:46,238 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2a3afbaa90414170a72fcdc0f825ab24, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/5ba279c40ba34c42914d1c48abc7b206, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ae85fdc32b774842b6765d9bcbfa05be] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.4 K 2024-11-10T18:14:46,239 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a3afbaa90414170a72fcdc0f825ab24, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262482497 2024-11-10T18:14:46,239 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ba279c40ba34c42914d1c48abc7b206, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262483319 2024-11-10T18:14:46,240 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae85fdc32b774842b6765d9bcbfa05be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731262484518 2024-11-10T18:14:46,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/31f8ad6651954a80adaaf78852ade8de is 50, key is test_row_0/B:col10/1731262484594/Put/seqid=0 2024-11-10T18:14:46,246 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:46,247 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/3efebc89ddf847a8a4b3cce0d2a501e1 is 50, key is test_row_0/C:col10/1731262484518/Put/seqid=0 2024-11-10T18:14:46,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742118_1294 (size=12301) 2024-11-10T18:14:46,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742119_1295 (size=12915) 2024-11-10T18:14:46,667 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/31f8ad6651954a80adaaf78852ade8de 2024-11-10T18:14:46,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/f995bfc476b944569f0298bcd3a93428 is 50, key is test_row_0/C:col10/1731262484594/Put/seqid=0 2024-11-10T18:14:46,679 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/3efebc89ddf847a8a4b3cce0d2a501e1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/3efebc89ddf847a8a4b3cce0d2a501e1 2024-11-10T18:14:46,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742120_1296 (size=12301) 2024-11-10T18:14:46,685 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into 3efebc89ddf847a8a4b3cce0d2a501e1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:46,685 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:46,685 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262485802; duration=0sec 2024-11-10T18:14:46,685 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:46,685 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:46,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:46,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:46,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262546829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:46,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262546934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,083 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/f995bfc476b944569f0298bcd3a93428 2024-11-10T18:14:47,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/2b9a5252e50544828d3ad409d005c5b7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/2b9a5252e50544828d3ad409d005c5b7 2024-11-10T18:14:47,091 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/2b9a5252e50544828d3ad409d005c5b7, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T18:14:47,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/31f8ad6651954a80adaaf78852ade8de as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/31f8ad6651954a80adaaf78852ade8de 2024-11-10T18:14:47,096 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/31f8ad6651954a80adaaf78852ade8de, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T18:14:47,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/f995bfc476b944569f0298bcd3a93428 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/f995bfc476b944569f0298bcd3a93428 2024-11-10T18:14:47,100 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/f995bfc476b944569f0298bcd3a93428, entries=150, sequenceid=317, filesize=12.0 K 2024-11-10T18:14:47,101 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8db0c68c9c5ead4c454abb5a20de2443 in 1281ms, sequenceid=317, compaction requested=false 2024-11-10T18:14:47,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:47,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:47,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-10T18:14:47,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-10T18:14:47,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-10T18:14:47,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-11-10T18:14:47,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.3590 sec 2024-11-10T18:14:47,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:47,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:14:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:47,149 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:47,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/36595e51b51245e7809c4143e69ac7b9 is 50, key is test_row_0/A:col10/1731262486799/Put/seqid=0 2024-11-10T18:14:47,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742121_1297 (size=12301) 2024-11-10T18:14:47,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262547230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,236 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262547344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,348 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,349 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262547554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,559 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/36595e51b51245e7809c4143e69ac7b9 2024-11-10T18:14:47,576 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/7ce4cae8d1cd4593aea126d2fb8eee2b is 50, key is test_row_0/B:col10/1731262486799/Put/seqid=0 2024-11-10T18:14:47,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42484 deadline: 1731262547574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742122_1298 (size=12301) 2024-11-10T18:14:47,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/7ce4cae8d1cd4593aea126d2fb8eee2b 2024-11-10T18:14:47,580 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,580 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/41b43359936446bf82fb99334c2e11fb is 50, key is test_row_0/C:col10/1731262486799/Put/seqid=0 2024-11-10T18:14:47,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742123_1299 (size=12301) 2024-11-10T18:14:47,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42442 deadline: 1731262547674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,679 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42510 deadline: 1731262547685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,689 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,709 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42508 deadline: 1731262547705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,709 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 8db0c68c9c5ead4c454abb5a20de2443 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:47,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262547864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:47,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:47,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/41b43359936446bf82fb99334c2e11fb 2024-11-10T18:14:47,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/36595e51b51245e7809c4143e69ac7b9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/36595e51b51245e7809c4143e69ac7b9 2024-11-10T18:14:47,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/36595e51b51245e7809c4143e69ac7b9, entries=150, sequenceid=333, filesize=12.0 K 2024-11-10T18:14:47,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/7ce4cae8d1cd4593aea126d2fb8eee2b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/7ce4cae8d1cd4593aea126d2fb8eee2b 2024-11-10T18:14:48,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/7ce4cae8d1cd4593aea126d2fb8eee2b, entries=150, sequenceid=333, filesize=12.0 K 2024-11-10T18:14:48,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/41b43359936446bf82fb99334c2e11fb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/41b43359936446bf82fb99334c2e11fb 2024-11-10T18:14:48,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/41b43359936446bf82fb99334c2e11fb, entries=150, sequenceid=333, filesize=12.0 K 2024-11-10T18:14:48,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 8db0c68c9c5ead4c454abb5a20de2443 in 860ms, sequenceid=333, compaction requested=true 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:48,008 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:48,008 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:48,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:48,009 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:48,009 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:48,009 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:48,009 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:48,009 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:48,009 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:48,009 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3288babfb6734559afd922515bd01f26, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/31f8ad6651954a80adaaf78852ade8de, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/7ce4cae8d1cd4593aea126d2fb8eee2b] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.6 K 2024-11-10T18:14:48,009 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/40d3ddc6e33f4aba888c5faf5da340c3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/2b9a5252e50544828d3ad409d005c5b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/36595e51b51245e7809c4143e69ac7b9] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.6 K 2024-11-10T18:14:48,010 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3288babfb6734559afd922515bd01f26, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731262484518 2024-11-10T18:14:48,010 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40d3ddc6e33f4aba888c5faf5da340c3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731262484518 2024-11-10T18:14:48,010 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b9a5252e50544828d3ad409d005c5b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731262484594 2024-11-10T18:14:48,010 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f8ad6651954a80adaaf78852ade8de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731262484594 2024-11-10T18:14:48,010 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36595e51b51245e7809c4143e69ac7b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731262486799 2024-11-10T18:14:48,010 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ce4cae8d1cd4593aea126d2fb8eee2b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731262486799 2024-11-10T18:14:48,023 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:48,023 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e17348fa8f1d4b4a8dce7c83e4028ac4 is 50, key is test_row_0/B:col10/1731262486799/Put/seqid=0 2024-11-10T18:14:48,026 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#256 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:48,027 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/fad8ff22d5c14357b78bfce2ffd4ac0f is 50, key is test_row_0/A:col10/1731262486799/Put/seqid=0 2024-11-10T18:14:48,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742124_1300 (size=13017) 2024-11-10T18:14:48,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742125_1301 (size=13017) 2024-11-10T18:14:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:48,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:14:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:48,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:48,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6b09f8395af54bac83e5316a1e224822 is 50, key is test_row_0/A:col10/1731262487223/Put/seqid=0 2024-11-10T18:14:48,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742126_1302 (size=12301) 2024-11-10T18:14:48,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262548419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:48,424 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:48,443 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e17348fa8f1d4b4a8dce7c83e4028ac4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e17348fa8f1d4b4a8dce7c83e4028ac4 2024-11-10T18:14:48,456 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/fad8ff22d5c14357b78bfce2ffd4ac0f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fad8ff22d5c14357b78bfce2ffd4ac0f 2024-11-10T18:14:48,457 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into e17348fa8f1d4b4a8dce7c83e4028ac4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:48,457 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:48,457 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=13, startTime=1731262488008; duration=0sec 2024-11-10T18:14:48,457 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:48,457 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:48,457 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:14:48,459 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:14:48,459 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:48,459 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:48,459 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/3efebc89ddf847a8a4b3cce0d2a501e1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/f995bfc476b944569f0298bcd3a93428, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/41b43359936446bf82fb99334c2e11fb] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=36.6 K 2024-11-10T18:14:48,460 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3efebc89ddf847a8a4b3cce0d2a501e1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731262484518 2024-11-10T18:14:48,460 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f995bfc476b944569f0298bcd3a93428, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1731262484594 2024-11-10T18:14:48,460 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 41b43359936446bf82fb99334c2e11fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731262486799 2024-11-10T18:14:48,461 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into fad8ff22d5c14357b78bfce2ffd4ac0f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:48,461 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:48,461 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=13, startTime=1731262488008; duration=0sec 2024-11-10T18:14:48,462 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:48,462 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:48,468 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:48,468 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/51202aba26614ed9b10dccc80f18d69f is 50, key is test_row_0/C:col10/1731262486799/Put/seqid=0 2024-11-10T18:14:48,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742127_1303 (size=13017) 2024-11-10T18:14:48,476 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/51202aba26614ed9b10dccc80f18d69f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/51202aba26614ed9b10dccc80f18d69f 2024-11-10T18:14:48,481 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into 51202aba26614ed9b10dccc80f18d69f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:48,482 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:48,482 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=13, startTime=1731262488008; duration=0sec 2024-11-10T18:14:48,482 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:48,482 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:48,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:48,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262548535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:48,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:48,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262548744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:48,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:48,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6b09f8395af54bac83e5316a1e224822 2024-11-10T18:14:48,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3900ae48fb0140f2ad9d0cb3dba3670a is 50, key is test_row_0/B:col10/1731262487223/Put/seqid=0 2024-11-10T18:14:48,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742128_1304 (size=12301) 2024-11-10T18:14:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-10T18:14:48,925 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:48,925 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-10T18:14:48,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T18:14:48,928 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:48,928 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:48,928 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:49,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T18:14:49,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:49,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262549055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:49,056 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:49,057 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:49,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T18:14:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3900ae48fb0140f2ad9d0cb3dba3670a 2024-11-10T18:14:49,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/1fcacf4f425346218d5d46f1fe817366 is 50, key is test_row_0/C:col10/1731262487223/Put/seqid=0 2024-11-10T18:14:49,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742129_1305 (size=12301) 2024-11-10T18:14:49,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T18:14:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T18:14:49,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T18:14:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T18:14:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T18:14:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262549565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:49,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=353 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/1fcacf4f425346218d5d46f1fe817366 2024-11-10T18:14:49,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6b09f8395af54bac83e5316a1e224822 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6b09f8395af54bac83e5316a1e224822 2024-11-10T18:14:49,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6b09f8395af54bac83e5316a1e224822, entries=150, sequenceid=353, filesize=12.0 K 2024-11-10T18:14:49,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/3900ae48fb0140f2ad9d0cb3dba3670a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3900ae48fb0140f2ad9d0cb3dba3670a 2024-11-10T18:14:49,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3900ae48fb0140f2ad9d0cb3dba3670a, entries=150, sequenceid=353, filesize=12.0 K 2024-11-10T18:14:49,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/1fcacf4f425346218d5d46f1fe817366 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fcacf4f425346218d5d46f1fe817366 2024-11-10T18:14:49,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fcacf4f425346218d5d46f1fe817366, entries=150, sequenceid=353, filesize=12.0 K 2024-11-10T18:14:49,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 8db0c68c9c5ead4c454abb5a20de2443 in 1251ms, sequenceid=353, compaction requested=false 2024-11-10T18:14:49,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:49,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-10T18:14:49,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:49,693 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:49,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:49,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e65ad53fdf3143989f0ffed85215f594 is 50, key is test_row_0/A:col10/1731262488417/Put/seqid=0 2024-11-10T18:14:49,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742130_1306 (size=9857) 2024-11-10T18:14:49,712 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e65ad53fdf3143989f0ffed85215f594 2024-11-10T18:14:49,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/13b7fc31888e444c9f9ba1e6aec45baa is 50, key is test_row_0/B:col10/1731262488417/Put/seqid=0 2024-11-10T18:14:49,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742131_1307 (size=9857) 2024-11-10T18:14:49,724 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/13b7fc31888e444c9f9ba1e6aec45baa 2024-11-10T18:14:49,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/92e1878efc4642f9b823f29ab6a3b581 is 50, key is test_row_0/C:col10/1731262488417/Put/seqid=0 2024-11-10T18:14:49,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742132_1308 (size=9857) 2024-11-10T18:14:50,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T18:14:50,135 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/92e1878efc4642f9b823f29ab6a3b581 2024-11-10T18:14:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/e65ad53fdf3143989f0ffed85215f594 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e65ad53fdf3143989f0ffed85215f594 2024-11-10T18:14:50,144 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e65ad53fdf3143989f0ffed85215f594, entries=100, sequenceid=368, filesize=9.6 K 2024-11-10T18:14:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/13b7fc31888e444c9f9ba1e6aec45baa as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/13b7fc31888e444c9f9ba1e6aec45baa 2024-11-10T18:14:50,149 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/13b7fc31888e444c9f9ba1e6aec45baa, entries=100, sequenceid=368, filesize=9.6 K 2024-11-10T18:14:50,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/92e1878efc4642f9b823f29ab6a3b581 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/92e1878efc4642f9b823f29ab6a3b581 2024-11-10T18:14:50,153 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/92e1878efc4642f9b823f29ab6a3b581, entries=100, sequenceid=368, filesize=9.6 K 2024-11-10T18:14:50,154 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 8db0c68c9c5ead4c454abb5a20de2443 in 461ms, sequenceid=368, compaction requested=true 2024-11-10T18:14:50,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:50,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:50,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-10T18:14:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-10T18:14:50,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-10T18:14:50,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2270 sec 2024-11-10T18:14:50,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.2320 sec 2024-11-10T18:14:50,493 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T18:14:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:50,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:50,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:50,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:50,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:50,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:50,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:50,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:50,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/a797fbbaf63d441392a43da0747eb926 is 50, key is test_row_0/A:col10/1731262490578/Put/seqid=0 2024-11-10T18:14:50,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742133_1309 (size=14741) 2024-11-10T18:14:50,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262490645, max timestamp=1731262490682Num uniques 859; 2024-11-10T18:14:50,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:50,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262490645, max timestamp=1731262490682Num uniques 859; 2024-11-10T18:14:50,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262490645, max timestamp=1731262490682Num uniques 859; 2024-11-10T18:14:50,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:50,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:50,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:50,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262550781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:50,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262550895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:50,899 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:50,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/a797fbbaf63d441392a43da0747eb926 2024-11-10T18:14:51,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d85f8a0114094153af903c3ca29ac9cd is 50, key is test_row_0/B:col10/1731262490578/Put/seqid=0 2024-11-10T18:14:51,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742134_1310 (size=12301) 2024-11-10T18:14:51,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d85f8a0114094153af903c3ca29ac9cd 2024-11-10T18:14:51,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/d6654f6cfb664cd5a6af8d4820e86932 is 50, key is test_row_0/C:col10/1731262490578/Put/seqid=0 2024-11-10T18:14:51,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742135_1311 (size=12301) 2024-11-10T18:14:51,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/d6654f6cfb664cd5a6af8d4820e86932 2024-11-10T18:14:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/a797fbbaf63d441392a43da0747eb926 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a797fbbaf63d441392a43da0747eb926 2024-11-10T18:14:51,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a797fbbaf63d441392a43da0747eb926, entries=200, sequenceid=378, filesize=14.4 K 2024-11-10T18:14:51,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/d85f8a0114094153af903c3ca29ac9cd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d85f8a0114094153af903c3ca29ac9cd 2024-11-10T18:14:51,032 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d85f8a0114094153af903c3ca29ac9cd, entries=150, sequenceid=378, filesize=12.0 K 2024-11-10T18:14:51,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/d6654f6cfb664cd5a6af8d4820e86932 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/d6654f6cfb664cd5a6af8d4820e86932 2024-11-10T18:14:51,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/d6654f6cfb664cd5a6af8d4820e86932, entries=150, sequenceid=378, filesize=12.0 K 2024-11-10T18:14:51,037 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 8db0c68c9c5ead4c454abb5a20de2443 in 448ms, sequenceid=378, compaction requested=true 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:51,037 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:51,037 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:51,037 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:51,038 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49916 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:51,038 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:51,038 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:51,039 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:51,039 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,039 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,039 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fad8ff22d5c14357b78bfce2ffd4ac0f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6b09f8395af54bac83e5316a1e224822, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e65ad53fdf3143989f0ffed85215f594, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a797fbbaf63d441392a43da0747eb926] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=48.7 K 2024-11-10T18:14:51,039 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e17348fa8f1d4b4a8dce7c83e4028ac4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3900ae48fb0140f2ad9d0cb3dba3670a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/13b7fc31888e444c9f9ba1e6aec45baa, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d85f8a0114094153af903c3ca29ac9cd] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=46.4 K 2024-11-10T18:14:51,039 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e17348fa8f1d4b4a8dce7c83e4028ac4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731262486799 2024-11-10T18:14:51,039 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting fad8ff22d5c14357b78bfce2ffd4ac0f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731262486799 2024-11-10T18:14:51,039 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b09f8395af54bac83e5316a1e224822, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731262487223 2024-11-10T18:14:51,039 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3900ae48fb0140f2ad9d0cb3dba3670a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731262487223 2024-11-10T18:14:51,040 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e65ad53fdf3143989f0ffed85215f594, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262488407 2024-11-10T18:14:51,040 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b7fc31888e444c9f9ba1e6aec45baa, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262488407 2024-11-10T18:14:51,040 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting a797fbbaf63d441392a43da0747eb926, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731262490578 2024-11-10T18:14:51,040 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d85f8a0114094153af903c3ca29ac9cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731262490578 2024-11-10T18:14:51,051 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:51,051 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/96af1828d57a4c0fa951eca8d5aa59dc is 50, key is test_row_0/B:col10/1731262490578/Put/seqid=0 2024-11-10T18:14:51,051 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#267 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:51,052 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/3561f68f871b4dc5850f9c3b87656e37 is 50, key is test_row_0/A:col10/1731262490578/Put/seqid=0 2024-11-10T18:14:51,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742136_1312 (size=13153) 2024-11-10T18:14:51,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742137_1313 (size=13153) 2024-11-10T18:14:51,066 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/96af1828d57a4c0fa951eca8d5aa59dc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/96af1828d57a4c0fa951eca8d5aa59dc 2024-11-10T18:14:51,071 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into 96af1828d57a4c0fa951eca8d5aa59dc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:51,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:51,071 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=12, startTime=1731262491037; duration=0sec 2024-11-10T18:14:51,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:51,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:51,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:51,072 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:51,072 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:51,072 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,072 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/51202aba26614ed9b10dccc80f18d69f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fcacf4f425346218d5d46f1fe817366, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/92e1878efc4642f9b823f29ab6a3b581, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/d6654f6cfb664cd5a6af8d4820e86932] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=46.4 K 2024-11-10T18:14:51,073 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 51202aba26614ed9b10dccc80f18d69f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=333, earliestPutTs=1731262486799 2024-11-10T18:14:51,073 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fcacf4f425346218d5d46f1fe817366, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=353, earliestPutTs=1731262487223 2024-11-10T18:14:51,073 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 92e1878efc4642f9b823f29ab6a3b581, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1731262488407 2024-11-10T18:14:51,074 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d6654f6cfb664cd5a6af8d4820e86932, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731262490578 2024-11-10T18:14:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-10T18:14:51,075 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:51,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:14:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-10T18:14:51,078 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:14:51,079 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:14:51,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:14:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:51,082 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:51,083 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2d5697697a484c55971de57a630bbceb is 50, key is test_row_0/C:col10/1731262490578/Put/seqid=0 2024-11-10T18:14:51,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742138_1314 (size=13153) 2024-11-10T18:14:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:51,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-10T18:14:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T18:14:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T18:14:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T18:14:51,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/1ddade00f0a04493b26dc922daf1f0ae is 50, key is test_row_0/A:col10/1731262490772/Put/seqid=0 2024-11-10T18:14:51,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742139_1315 (size=12301) 2024-11-10T18:14:51,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/1ddade00f0a04493b26dc922daf1f0ae 2024-11-10T18:14:51,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e3ec409e4ce5471d9c0ebec58438ba63 is 50, key is test_row_0/B:col10/1731262490772/Put/seqid=0 2024-11-10T18:14:51,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742140_1316 (size=12301) 2024-11-10T18:14:51,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262551147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,154 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:51,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T18:14:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 267 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262551265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:51,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T18:14:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:51,461 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/3561f68f871b4dc5850f9c3b87656e37 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/3561f68f871b4dc5850f9c3b87656e37 2024-11-10T18:14:51,465 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into 3561f68f871b4dc5850f9c3b87656e37(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:51,465 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:51,465 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=12, startTime=1731262491037; duration=0sec 2024-11-10T18:14:51,465 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:51,465 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:51,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262551475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:51,492 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2d5697697a484c55971de57a630bbceb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2d5697697a484c55971de57a630bbceb 2024-11-10T18:14:51,497 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into 2d5697697a484c55971de57a630bbceb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:51,497 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:51,497 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=12, startTime=1731262491037; duration=0sec 2024-11-10T18:14:51,497 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:51,497 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:51,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T18:14:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e3ec409e4ce5471d9c0ebec58438ba63 2024-11-10T18:14:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2f42195380f4416cb200fe9865b5220d is 50, key is test_row_0/C:col10/1731262490772/Put/seqid=0 2024-11-10T18:14:51,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742141_1317 (size=12301) 2024-11-10T18:14:51,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T18:14:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:51,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262551784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:51,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:51,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T18:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:14:51,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2f42195380f4416cb200fe9865b5220d 2024-11-10T18:14:51,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/1ddade00f0a04493b26dc922daf1f0ae as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/1ddade00f0a04493b26dc922daf1f0ae 2024-11-10T18:14:51,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/1ddade00f0a04493b26dc922daf1f0ae, entries=150, sequenceid=417, filesize=12.0 K 2024-11-10T18:14:51,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/e3ec409e4ce5471d9c0ebec58438ba63 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e3ec409e4ce5471d9c0ebec58438ba63 2024-11-10T18:14:51,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e3ec409e4ce5471d9c0ebec58438ba63, entries=150, sequenceid=417, filesize=12.0 K 2024-11-10T18:14:51,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/2f42195380f4416cb200fe9865b5220d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2f42195380f4416cb200fe9865b5220d 2024-11-10T18:14:51,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2f42195380f4416cb200fe9865b5220d, entries=150, sequenceid=417, filesize=12.0 K 2024-11-10T18:14:51,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 8db0c68c9c5ead4c454abb5a20de2443 in 859ms, sequenceid=417, compaction requested=false 2024-11-10T18:14:51,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:51,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-10T18:14:51,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:51,998 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T18:14:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:52,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6adbed2626a7489c8ff919c8edce8d1f is 50, key is test_row_0/A:col10/1731262491139/Put/seqid=0 2024-11-10T18:14:52,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742142_1318 (size=12301) 2024-11-10T18:14:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:52,298 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. as already flushing 2024-11-10T18:14:52,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262492344, max timestamp=1731262492368Num uniques 859; 2024-11-10T18:14:52,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:52,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262492344, max timestamp=1731262492368Num uniques 859; 2024-11-10T18:14:52,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:52,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262492344, max timestamp=1731262492368Num uniques 859; 2024-11-10T18:14:52,371 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:14:52,407 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6adbed2626a7489c8ff919c8edce8d1f 2024-11-10T18:14:52,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/ce407246a9b543b9b2c0a22fe383c0a9 is 50, key is test_row_0/B:col10/1731262491139/Put/seqid=0 2024-11-10T18:14:52,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742143_1319 (size=12301) 2024-11-10T18:14:52,480 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731262492456, max timestamp=1731262492476Num uniques 1018; 2024-11-10T18:14:52,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T18:14:52,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731262492456, max timestamp=1731262492476Num uniques 1018; 2024-11-10T18:14:52,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T18:14:52,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731262492456, max timestamp=1731262492476Num uniques 1018; 2024-11-10T18:14:52,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-10T18:14:52,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731262492548, max timestamp=1731262492573Num uniques 1027; 2024-11-10T18:14:52,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T18:14:52,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731262492548, max timestamp=1731262492573Num uniques 1027; 2024-11-10T18:14:52,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731262492548, max timestamp=1731262492573Num uniques 1027; 2024-11-10T18:14:52,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T18:14:52,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-10T18:14:52,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731262492642, max timestamp=1731262492675Num uniques 1036; 2024-11-10T18:14:52,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-10T18:14:52,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731262492642, max timestamp=1731262492675Num uniques 1036; 2024-11-10T18:14:52,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-10T18:14:52,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731262492642, max timestamp=1731262492675Num uniques 1036; 2024-11-10T18:14:52,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-10T18:14:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731262492741, max timestamp=1731262492771Num uniques 1045; 2024-11-10T18:14:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731262492741, max timestamp=1731262492771Num uniques 1045; 2024-11-10T18:14:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-10T18:14:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-10T18:14:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731262492741, max timestamp=1731262492771Num uniques 1045; 2024-11-10T18:14:52,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-10T18:14:52,819 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/ce407246a9b543b9b2c0a22fe383c0a9 2024-11-10T18:14:52,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/6445f6dcdc1f4ceeba0cc63ce384ce55 is 50, key is test_row_0/C:col10/1731262491139/Put/seqid=0 2024-11-10T18:14:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742144_1320 (size=12301) 2024-11-10T18:14:52,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 373 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262552880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:52,887 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:52,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:52,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 374 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262552995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:52,999 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:53,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:53,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 375 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262553205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:53,210 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:53,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:53,237 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/6445f6dcdc1f4ceeba0cc63ce384ce55 2024-11-10T18:14:53,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/6adbed2626a7489c8ff919c8edce8d1f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6adbed2626a7489c8ff919c8edce8d1f 2024-11-10T18:14:53,245 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6adbed2626a7489c8ff919c8edce8d1f, entries=150, sequenceid=428, filesize=12.0 K 2024-11-10T18:14:53,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/ce407246a9b543b9b2c0a22fe383c0a9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/ce407246a9b543b9b2c0a22fe383c0a9 2024-11-10T18:14:53,250 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/ce407246a9b543b9b2c0a22fe383c0a9, entries=150, sequenceid=428, filesize=12.0 K 2024-11-10T18:14:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/6445f6dcdc1f4ceeba0cc63ce384ce55 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/6445f6dcdc1f4ceeba0cc63ce384ce55 2024-11-10T18:14:53,256 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/6445f6dcdc1f4ceeba0cc63ce384ce55, entries=150, sequenceid=428, filesize=12.0 K 2024-11-10T18:14:53,258 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 8db0c68c9c5ead4c454abb5a20de2443 in 1260ms, sequenceid=428, compaction requested=true 2024-11-10T18:14:53,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:53,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:53,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-10T18:14:53,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-10T18:14:53,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-10T18:14:53,263 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1820 sec 2024-11-10T18:14:53,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.1880 sec 2024-11-10T18:14:53,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:53,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=147.60 KB heapSize=422.16 KB 2024-11-10T18:14:53,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:53,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T18:14:53,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T18:14:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:53,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-10T18:14:53,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c50da68d48e5497f830f51bbadb50c6d is 50, key is test_row_0/A:col10/1731262492878/Put/seqid=0 2024-11-10T18:14:53,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742145_1321 (size=14741) 2024-11-10T18:14:53,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 382 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262553546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:53,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 383 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262553654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:53,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:53,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 384 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262553865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:53,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:53,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c50da68d48e5497f830f51bbadb50c6d 2024-11-10T18:14:53,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/61357d3e78954a43a5e189f364242f92 is 50, key is test_row_0/B:col10/1731262492878/Put/seqid=0 2024-11-10T18:14:53,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742146_1322 (size=12301) 2024-11-10T18:14:53,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/61357d3e78954a43a5e189f364242f92 2024-11-10T18:14:53,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7363eb7b4f5046c49a043a8eab3ec637 is 50, key is test_row_0/C:col10/1731262492878/Put/seqid=0 2024-11-10T18:14:53,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742147_1323 (size=12301) 2024-11-10T18:14:54,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:14:54,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 385 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:42468 deadline: 1731262554175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 2024-11-10T18:14:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:14:54,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:14:54,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7363eb7b4f5046c49a043a8eab3ec637 2024-11-10T18:14:54,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c50da68d48e5497f830f51bbadb50c6d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c50da68d48e5497f830f51bbadb50c6d 2024-11-10T18:14:54,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c50da68d48e5497f830f51bbadb50c6d, entries=200, sequenceid=535, filesize=14.4 K 2024-11-10T18:14:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/61357d3e78954a43a5e189f364242f92 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/61357d3e78954a43a5e189f364242f92 2024-11-10T18:14:54,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/61357d3e78954a43a5e189f364242f92, entries=150, sequenceid=535, filesize=12.0 K 2024-11-10T18:14:54,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7363eb7b4f5046c49a043a8eab3ec637 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7363eb7b4f5046c49a043a8eab3ec637 2024-11-10T18:14:54,369 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-10T18:14:54,369 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:54,369 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,369 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,369 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:54,371 INFO [Thread-1118 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1118. 2024-11-10T18:14:54,371 DEBUG [Thread-1118 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:54,371 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,371 DEBUG [Thread-1118 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,371 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-10T18:14:54,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:54,371 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:54,371 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,371 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:54,372 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-10T18:14:54,373 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:54,373 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,373 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:54,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7363eb7b4f5046c49a043a8eab3ec637, entries=150, sequenceid=535, filesize=12.0 K 2024-11-10T18:14:54,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~422.11 KB/432240, currentSize=33.54 KB/34350 for 8db0c68c9c5ead4c454abb5a20de2443 in 858ms, sequenceid=535, compaction requested=true 2024-11-10T18:14:54,375 INFO [Thread-1116 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1116. 2024-11-10T18:14:54,375 DEBUG [Thread-1116 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:54,375 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,375 DEBUG [Thread-1116 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:14:54,375 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:54,375 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8db0c68c9c5ead4c454abb5a20de2443:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:14:54,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:54,376 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:54,376 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:54,376 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/B is initiating minor compaction (all files) 2024-11-10T18:14:54,376 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/A is initiating minor compaction (all files) 2024-11-10T18:14:54,376 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/B in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:54,376 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/A in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:54,376 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/96af1828d57a4c0fa951eca8d5aa59dc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e3ec409e4ce5471d9c0ebec58438ba63, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/ce407246a9b543b9b2c0a22fe383c0a9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/61357d3e78954a43a5e189f364242f92] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=48.9 K 2024-11-10T18:14:54,376 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/3561f68f871b4dc5850f9c3b87656e37, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/1ddade00f0a04493b26dc922daf1f0ae, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6adbed2626a7489c8ff919c8edce8d1f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c50da68d48e5497f830f51bbadb50c6d] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=51.3 K 2024-11-10T18:14:54,376 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96af1828d57a4c0fa951eca8d5aa59dc, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731262490578 2024-11-10T18:14:54,376 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3561f68f871b4dc5850f9c3b87656e37, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731262490578 2024-11-10T18:14:54,377 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3ec409e4ce5471d9c0ebec58438ba63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731262490772 2024-11-10T18:14:54,377 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ddade00f0a04493b26dc922daf1f0ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731262490772 2024-11-10T18:14:54,377 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce407246a9b543b9b2c0a22fe383c0a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731262491128 2024-11-10T18:14:54,377 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6adbed2626a7489c8ff919c8edce8d1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731262491128 2024-11-10T18:14:54,377 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 61357d3e78954a43a5e189f364242f92, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731262492871 2024-11-10T18:14:54,377 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c50da68d48e5497f830f51bbadb50c6d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731262492838 2024-11-10T18:14:54,383 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#A#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:54,384 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c0aef3790c7644fc830d6e65f0a88513 is 50, key is test_row_0/A:col10/1731262492878/Put/seqid=0 2024-11-10T18:14:54,386 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:54,386 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/8983a49d667c489fa8c8d2919613ed8a is 50, key is test_row_0/B:col10/1731262492878/Put/seqid=0 2024-11-10T18:14:54,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742148_1324 (size=13289) 2024-11-10T18:14:54,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742149_1325 (size=13289) 2024-11-10T18:14:54,686 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-10T18:14:54,686 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:54,686 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,686 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:54,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:54,793 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/c0aef3790c7644fc830d6e65f0a88513 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c0aef3790c7644fc830d6e65f0a88513 2024-11-10T18:14:54,793 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/8983a49d667c489fa8c8d2919613ed8a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8983a49d667c489fa8c8d2919613ed8a 2024-11-10T18:14:54,797 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/B of 8db0c68c9c5ead4c454abb5a20de2443 into 8983a49d667c489fa8c8d2919613ed8a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:54,797 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/A of 8db0c68c9c5ead4c454abb5a20de2443 into c0aef3790c7644fc830d6e65f0a88513(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:54,797 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/B, priority=12, startTime=1731262494375; duration=0sec 2024-11-10T18:14:54,797 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/A, priority=12, startTime=1731262494375; duration=0sec 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:B 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:A 2024-11-10T18:14:54,797 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:14:54,798 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:14:54,798 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 8db0c68c9c5ead4c454abb5a20de2443/C is initiating minor compaction (all files) 2024-11-10T18:14:54,798 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8db0c68c9c5ead4c454abb5a20de2443/C in TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:54,798 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2d5697697a484c55971de57a630bbceb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2f42195380f4416cb200fe9865b5220d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/6445f6dcdc1f4ceeba0cc63ce384ce55, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7363eb7b4f5046c49a043a8eab3ec637] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp, totalSize=48.9 K 2024-11-10T18:14:54,798 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d5697697a484c55971de57a630bbceb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731262490578 2024-11-10T18:14:54,799 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f42195380f4416cb200fe9865b5220d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731262490772 2024-11-10T18:14:54,799 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6445f6dcdc1f4ceeba0cc63ce384ce55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1731262491128 2024-11-10T18:14:54,799 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7363eb7b4f5046c49a043a8eab3ec637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=535, earliestPutTs=1731262492871 2024-11-10T18:14:54,805 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8db0c68c9c5ead4c454abb5a20de2443#C#compaction#281 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:14:54,806 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ad51b4f653724f848598a8402408ab48 is 50, key is test_row_0/C:col10/1731262492878/Put/seqid=0 2024-11-10T18:14:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742150_1326 (size=13289) 2024-11-10T18:14:55,213 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/ad51b4f653724f848598a8402408ab48 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ad51b4f653724f848598a8402408ab48 2024-11-10T18:14:55,217 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8db0c68c9c5ead4c454abb5a20de2443/C of 8db0c68c9c5ead4c454abb5a20de2443 into ad51b4f653724f848598a8402408ab48(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:14:55,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:55,217 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., storeName=8db0c68c9c5ead4c454abb5a20de2443/C, priority=12, startTime=1731262494375; duration=0sec 2024-11-10T18:14:55,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:14:55,217 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8db0c68c9c5ead4c454abb5a20de2443:C 2024-11-10T18:14:55,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-10T18:14:55,245 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:14:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:57,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:14:57,656 INFO [Thread-1107 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1107. 2024-11-10T18:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:57,656 DEBUG [Thread-1107 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:57,656 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:57,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:57,657 DEBUG [Thread-1107 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:57,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/be0787594d7346868cd9bf3307af1dea is 50, key is test_row_0/A:col10/1731262497655/Put/seqid=0 2024-11-10T18:14:57,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742151_1327 (size=9857) 2024-11-10T18:14:57,686 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-10T18:14:57,686 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:57,686 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,686 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:57,736 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-10T18:14:57,736 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:57,736 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,736 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,736 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:57,806 INFO [Thread-1105 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1105. 2024-11-10T18:14:57,806 DEBUG [Thread-1105 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:14:57,807 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,807 DEBUG [Thread-1105 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 36 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 28 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 330 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2973 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3031 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2919 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2981 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3043 2024-11-10T18:14:57,807 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T18:14:57,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:14:57,807 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:14:57,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:14:57,807 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:14:57,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T18:14:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T18:14:57,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:14:57,811 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262497811"}]},"ts":"1731262497811"} 2024-11-10T18:14:57,812 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T18:14:57,812 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T18:14:57,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:14:57,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, UNASSIGN}] 2024-11-10T18:14:57,815 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, UNASSIGN 2024-11-10T18:14:57,815 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8db0c68c9c5ead4c454abb5a20de2443, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:14:57,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, UNASSIGN because future has completed 2024-11-10T18:14:57,817 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:14:57,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:14:57,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:14:57,969 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:14:57,969 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:14:57,969 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 8db0c68c9c5ead4c454abb5a20de2443, disabling compactions & flushes 2024-11-10T18:14:57,969 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:58,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/be0787594d7346868cd9bf3307af1dea 2024-11-10T18:14:58,071 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/57719b21a28b4e32961c227bffefe57f is 50, key is test_row_0/B:col10/1731262497655/Put/seqid=0 2024-11-10T18:14:58,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742152_1328 (size=9857) 2024-11-10T18:14:58,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:14:58,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:14:58,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/57719b21a28b4e32961c227bffefe57f 2024-11-10T18:14:58,481 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/29a1eeefc94f4cd2a5977d239e67cc79 is 50, key is test_row_0/C:col10/1731262497655/Put/seqid=0 2024-11-10T18:14:58,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742153_1329 (size=9857) 2024-11-10T18:14:58,884 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/29a1eeefc94f4cd2a5977d239e67cc79 2024-11-10T18:14:58,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/be0787594d7346868cd9bf3307af1dea as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/be0787594d7346868cd9bf3307af1dea 2024-11-10T18:14:58,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/be0787594d7346868cd9bf3307af1dea, entries=100, sequenceid=548, filesize=9.6 K 2024-11-10T18:14:58,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/57719b21a28b4e32961c227bffefe57f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/57719b21a28b4e32961c227bffefe57f 2024-11-10T18:14:58,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/57719b21a28b4e32961c227bffefe57f, entries=100, sequenceid=548, filesize=9.6 K 2024-11-10T18:14:58,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/29a1eeefc94f4cd2a5977d239e67cc79 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/29a1eeefc94f4cd2a5977d239e67cc79 2024-11-10T18:14:58,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/29a1eeefc94f4cd2a5977d239e67cc79, entries=100, sequenceid=548, filesize=9.6 K 2024-11-10T18:14:58,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 8db0c68c9c5ead4c454abb5a20de2443 in 1242ms, sequenceid=548, compaction requested=false 2024-11-10T18:14:58,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8db0c68c9c5ead4c454abb5a20de2443: 2024-11-10T18:14:58,898 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. after waiting 0 ms 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:14:58,898 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 8db0c68c9c5ead4c454abb5a20de2443 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=A 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=B 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 8db0c68c9c5ead4c454abb5a20de2443, store=C 2024-11-10T18:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:14:58,901 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/f50630105d944a08adfd05397f928690 is 50, key is test_row_1/A:col10/1731262497685/Put/seqid=0 2024-11-10T18:14:58,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742154_1330 (size=9857) 2024-11-10T18:14:58,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:14:59,305 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/f50630105d944a08adfd05397f928690 2024-11-10T18:14:59,311 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/52e4d5e867664e78a2338c31331d808e is 50, key is test_row_1/B:col10/1731262497685/Put/seqid=0 2024-11-10T18:14:59,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742155_1331 (size=9857) 2024-11-10T18:14:59,715 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/52e4d5e867664e78a2338c31331d808e 2024-11-10T18:14:59,720 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7e8d4e3369e04e17b686f7dddb33a971 is 50, key is test_row_1/C:col10/1731262497685/Put/seqid=0 2024-11-10T18:14:59,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742156_1332 (size=9857) 2024-11-10T18:14:59,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:15:00,124 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=554 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7e8d4e3369e04e17b686f7dddb33a971 2024-11-10T18:15:00,128 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/A/f50630105d944a08adfd05397f928690 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f50630105d944a08adfd05397f928690 2024-11-10T18:15:00,131 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f50630105d944a08adfd05397f928690, entries=100, sequenceid=554, filesize=9.6 K 2024-11-10T18:15:00,132 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/B/52e4d5e867664e78a2338c31331d808e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e4d5e867664e78a2338c31331d808e 2024-11-10T18:15:00,135 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e4d5e867664e78a2338c31331d808e, entries=100, sequenceid=554, filesize=9.6 K 2024-11-10T18:15:00,135 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/.tmp/C/7e8d4e3369e04e17b686f7dddb33a971 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7e8d4e3369e04e17b686f7dddb33a971 2024-11-10T18:15:00,138 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7e8d4e3369e04e17b686f7dddb33a971, entries=100, sequenceid=554, filesize=9.6 K 2024-11-10T18:15:00,139 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 8db0c68c9c5ead4c454abb5a20de2443 in 1241ms, sequenceid=554, compaction requested=true 2024-11-10T18:15:00,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/7882aacec6604d4cba13412c1b32b8c6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/350c7a90e1de4df4ab7801a67f52a0af, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/86a810544bcd42a39609e0ee25daea5a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f010a1b8f57e4959900879a2ba5d95de, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/60c14841c1544bd6bebe0efbc9cec92a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c1919d3fedd14294b434e32ed2127469, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/b74f717f19b24a1e9fcb34139677cf7f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/df4f1c652e954268988165e8d5ffa50d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e09ebe3e27954b80878429eafb532a56, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/bc0a059c396e40febea2f8f00c310ed0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/07fe0187c78346debdcc51f96ce41d48, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6270f73d288a4121b60096119c5d1afc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e7662c8a94394c7ab98e3aa1723685b4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/83634311c86447b19183982583fd8667, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/d94c3fca66f54a03a1fd0fde4fea9db9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fc9a3979dec14acdbf8c9557e5602efa, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a9744642d52043ca855a1d58a542b668, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/4779a34bcbeb4299b7a1cc46063ff7fc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/40d3ddc6e33f4aba888c5faf5da340c3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/2b9a5252e50544828d3ad409d005c5b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fad8ff22d5c14357b78bfce2ffd4ac0f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/36595e51b51245e7809c4143e69ac7b9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6b09f8395af54bac83e5316a1e224822, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e65ad53fdf3143989f0ffed85215f594, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a797fbbaf63d441392a43da0747eb926, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/3561f68f871b4dc5850f9c3b87656e37, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/1ddade00f0a04493b26dc922daf1f0ae, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6adbed2626a7489c8ff919c8edce8d1f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c50da68d48e5497f830f51bbadb50c6d] to archive 2024-11-10T18:15:00,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:00,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/7882aacec6604d4cba13412c1b32b8c6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/7882aacec6604d4cba13412c1b32b8c6 2024-11-10T18:15:00,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/350c7a90e1de4df4ab7801a67f52a0af to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/350c7a90e1de4df4ab7801a67f52a0af 2024-11-10T18:15:00,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/86a810544bcd42a39609e0ee25daea5a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/86a810544bcd42a39609e0ee25daea5a 2024-11-10T18:15:00,144 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f010a1b8f57e4959900879a2ba5d95de to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f010a1b8f57e4959900879a2ba5d95de 2024-11-10T18:15:00,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/60c14841c1544bd6bebe0efbc9cec92a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/60c14841c1544bd6bebe0efbc9cec92a 2024-11-10T18:15:00,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c1919d3fedd14294b434e32ed2127469 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c1919d3fedd14294b434e32ed2127469 2024-11-10T18:15:00,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/b74f717f19b24a1e9fcb34139677cf7f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/b74f717f19b24a1e9fcb34139677cf7f 2024-11-10T18:15:00,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/df4f1c652e954268988165e8d5ffa50d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/df4f1c652e954268988165e8d5ffa50d 2024-11-10T18:15:00,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e09ebe3e27954b80878429eafb532a56 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e09ebe3e27954b80878429eafb532a56 2024-11-10T18:15:00,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/bc0a059c396e40febea2f8f00c310ed0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/bc0a059c396e40febea2f8f00c310ed0 2024-11-10T18:15:00,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/07fe0187c78346debdcc51f96ce41d48 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/07fe0187c78346debdcc51f96ce41d48 2024-11-10T18:15:00,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6270f73d288a4121b60096119c5d1afc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6270f73d288a4121b60096119c5d1afc 2024-11-10T18:15:00,151 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e7662c8a94394c7ab98e3aa1723685b4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e7662c8a94394c7ab98e3aa1723685b4 2024-11-10T18:15:00,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/83634311c86447b19183982583fd8667 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/83634311c86447b19183982583fd8667 2024-11-10T18:15:00,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/d94c3fca66f54a03a1fd0fde4fea9db9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/d94c3fca66f54a03a1fd0fde4fea9db9 2024-11-10T18:15:00,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fc9a3979dec14acdbf8c9557e5602efa to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fc9a3979dec14acdbf8c9557e5602efa 2024-11-10T18:15:00,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a9744642d52043ca855a1d58a542b668 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a9744642d52043ca855a1d58a542b668 2024-11-10T18:15:00,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/4779a34bcbeb4299b7a1cc46063ff7fc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/4779a34bcbeb4299b7a1cc46063ff7fc 2024-11-10T18:15:00,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/40d3ddc6e33f4aba888c5faf5da340c3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/40d3ddc6e33f4aba888c5faf5da340c3 2024-11-10T18:15:00,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/2b9a5252e50544828d3ad409d005c5b7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/2b9a5252e50544828d3ad409d005c5b7 2024-11-10T18:15:00,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fad8ff22d5c14357b78bfce2ffd4ac0f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/fad8ff22d5c14357b78bfce2ffd4ac0f 2024-11-10T18:15:00,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/36595e51b51245e7809c4143e69ac7b9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/36595e51b51245e7809c4143e69ac7b9 2024-11-10T18:15:00,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6b09f8395af54bac83e5316a1e224822 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6b09f8395af54bac83e5316a1e224822 2024-11-10T18:15:00,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e65ad53fdf3143989f0ffed85215f594 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/e65ad53fdf3143989f0ffed85215f594 2024-11-10T18:15:00,160 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a797fbbaf63d441392a43da0747eb926 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/a797fbbaf63d441392a43da0747eb926 2024-11-10T18:15:00,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/3561f68f871b4dc5850f9c3b87656e37 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/3561f68f871b4dc5850f9c3b87656e37 2024-11-10T18:15:00,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/1ddade00f0a04493b26dc922daf1f0ae to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/1ddade00f0a04493b26dc922daf1f0ae 2024-11-10T18:15:00,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6adbed2626a7489c8ff919c8edce8d1f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/6adbed2626a7489c8ff919c8edce8d1f 2024-11-10T18:15:00,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c50da68d48e5497f830f51bbadb50c6d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c50da68d48e5497f830f51bbadb50c6d 2024-11-10T18:15:00,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d1d4c2d28df4404bbbf3da7e7d16a88f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/5bd4f82e59ae4b7a82b69d3840a32064, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/09152455a07d4e278ae94ae17eb2ab91, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e96629da30416c883fd0e720bb0c6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/285f65bdd83f41f0beab966bffc6b13a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e2885791266c4fb9864090881c419cf9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/0144623e54fa4bee934d65f2f67902f9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/35ed70e4b92e407cb32a5f2f4a39b05d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8e814dcf09d146ed9de2a2501e4f4ed3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/eb7c32715b3946dab343935778e12867, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3a7c83e4e71c4fd6bcf9a875935c8990, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/af85ebeee4f04bf6818ce3334cca0e2f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d6f6c43bb7e64a01ae72840c6e6fa795, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/1552be142657461080a80652aea1ba76, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/08d44e0c6b5040b0ae9f88e14061bd9e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3bfd5ff749ee49b395a7820b3f51f384, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/cf074c20798f46a384ec853e45a8d120, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3288babfb6734559afd922515bd01f26, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3566e826513645c490129e86ce09d359, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/31f8ad6651954a80adaaf78852ade8de, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e17348fa8f1d4b4a8dce7c83e4028ac4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/7ce4cae8d1cd4593aea126d2fb8eee2b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3900ae48fb0140f2ad9d0cb3dba3670a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/13b7fc31888e444c9f9ba1e6aec45baa, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/96af1828d57a4c0fa951eca8d5aa59dc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d85f8a0114094153af903c3ca29ac9cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e3ec409e4ce5471d9c0ebec58438ba63, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/ce407246a9b543b9b2c0a22fe383c0a9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/61357d3e78954a43a5e189f364242f92] to archive 2024-11-10T18:15:00,165 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:00,166 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d1d4c2d28df4404bbbf3da7e7d16a88f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d1d4c2d28df4404bbbf3da7e7d16a88f 2024-11-10T18:15:00,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/5bd4f82e59ae4b7a82b69d3840a32064 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/5bd4f82e59ae4b7a82b69d3840a32064 2024-11-10T18:15:00,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/09152455a07d4e278ae94ae17eb2ab91 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/09152455a07d4e278ae94ae17eb2ab91 2024-11-10T18:15:00,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e96629da30416c883fd0e720bb0c6e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e96629da30416c883fd0e720bb0c6e 2024-11-10T18:15:00,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/285f65bdd83f41f0beab966bffc6b13a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/285f65bdd83f41f0beab966bffc6b13a 2024-11-10T18:15:00,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e2885791266c4fb9864090881c419cf9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e2885791266c4fb9864090881c419cf9 2024-11-10T18:15:00,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/0144623e54fa4bee934d65f2f67902f9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/0144623e54fa4bee934d65f2f67902f9 2024-11-10T18:15:00,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/35ed70e4b92e407cb32a5f2f4a39b05d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/35ed70e4b92e407cb32a5f2f4a39b05d 2024-11-10T18:15:00,173 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8e814dcf09d146ed9de2a2501e4f4ed3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8e814dcf09d146ed9de2a2501e4f4ed3 2024-11-10T18:15:00,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/eb7c32715b3946dab343935778e12867 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/eb7c32715b3946dab343935778e12867 2024-11-10T18:15:00,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3a7c83e4e71c4fd6bcf9a875935c8990 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3a7c83e4e71c4fd6bcf9a875935c8990 2024-11-10T18:15:00,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/af85ebeee4f04bf6818ce3334cca0e2f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/af85ebeee4f04bf6818ce3334cca0e2f 2024-11-10T18:15:00,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d6f6c43bb7e64a01ae72840c6e6fa795 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d6f6c43bb7e64a01ae72840c6e6fa795 2024-11-10T18:15:00,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/1552be142657461080a80652aea1ba76 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/1552be142657461080a80652aea1ba76 2024-11-10T18:15:00,178 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/08d44e0c6b5040b0ae9f88e14061bd9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/08d44e0c6b5040b0ae9f88e14061bd9e 2024-11-10T18:15:00,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3bfd5ff749ee49b395a7820b3f51f384 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3bfd5ff749ee49b395a7820b3f51f384 2024-11-10T18:15:00,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/cf074c20798f46a384ec853e45a8d120 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/cf074c20798f46a384ec853e45a8d120 2024-11-10T18:15:00,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3288babfb6734559afd922515bd01f26 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3288babfb6734559afd922515bd01f26 2024-11-10T18:15:00,182 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3566e826513645c490129e86ce09d359 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3566e826513645c490129e86ce09d359 2024-11-10T18:15:00,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/31f8ad6651954a80adaaf78852ade8de to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/31f8ad6651954a80adaaf78852ade8de 2024-11-10T18:15:00,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e17348fa8f1d4b4a8dce7c83e4028ac4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e17348fa8f1d4b4a8dce7c83e4028ac4 2024-11-10T18:15:00,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/7ce4cae8d1cd4593aea126d2fb8eee2b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/7ce4cae8d1cd4593aea126d2fb8eee2b 2024-11-10T18:15:00,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3900ae48fb0140f2ad9d0cb3dba3670a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/3900ae48fb0140f2ad9d0cb3dba3670a 2024-11-10T18:15:00,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/13b7fc31888e444c9f9ba1e6aec45baa to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/13b7fc31888e444c9f9ba1e6aec45baa 2024-11-10T18:15:00,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/96af1828d57a4c0fa951eca8d5aa59dc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/96af1828d57a4c0fa951eca8d5aa59dc 2024-11-10T18:15:00,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d85f8a0114094153af903c3ca29ac9cd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/d85f8a0114094153af903c3ca29ac9cd 2024-11-10T18:15:00,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e3ec409e4ce5471d9c0ebec58438ba63 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/e3ec409e4ce5471d9c0ebec58438ba63 2024-11-10T18:15:00,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/ce407246a9b543b9b2c0a22fe383c0a9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/ce407246a9b543b9b2c0a22fe383c0a9 2024-11-10T18:15:00,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/61357d3e78954a43a5e189f364242f92 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/61357d3e78954a43a5e189f364242f92 2024-11-10T18:15:00,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7243269aab3c478a9746b42b09fe6f46, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/b91d37c093d246f7b19eb21360577d69, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/63d3d249d18647cf87ddd51fdcc8ed18, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/03d7ff0b3ba949b99902649cbc9761b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/625c70f518944a47ad342fd59a25a1e6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/c6d071d8e91541d1aee54e8f65fe79fd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/829f285a22c947a29410d665713c74cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/34de94a6306e4d0b9c0104b791616399, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ce456e60ff9945eea7a67b1aa50744a6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/688249fe0d594d72a3a98d22a5c8d1ce, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/27a0d2e7ccb742de8c4dec32aa560c48, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fa2277b42654c6e80f93b9eba3f12a7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/003e36f9b6884e24a7e3afd7a2f87943, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/84faa428621640d7baea28d0ad27e088, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2a3afbaa90414170a72fcdc0f825ab24, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/010092ccdabe4fab99dd682b02e44a4f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/5ba279c40ba34c42914d1c48abc7b206, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/3efebc89ddf847a8a4b3cce0d2a501e1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ae85fdc32b774842b6765d9bcbfa05be, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/f995bfc476b944569f0298bcd3a93428, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/51202aba26614ed9b10dccc80f18d69f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/41b43359936446bf82fb99334c2e11fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fcacf4f425346218d5d46f1fe817366, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/92e1878efc4642f9b823f29ab6a3b581, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2d5697697a484c55971de57a630bbceb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/d6654f6cfb664cd5a6af8d4820e86932, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2f42195380f4416cb200fe9865b5220d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/6445f6dcdc1f4ceeba0cc63ce384ce55, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7363eb7b4f5046c49a043a8eab3ec637] to archive 2024-11-10T18:15:00,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:00,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7243269aab3c478a9746b42b09fe6f46 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7243269aab3c478a9746b42b09fe6f46 2024-11-10T18:15:00,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/b91d37c093d246f7b19eb21360577d69 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/b91d37c093d246f7b19eb21360577d69 2024-11-10T18:15:00,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/63d3d249d18647cf87ddd51fdcc8ed18 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/63d3d249d18647cf87ddd51fdcc8ed18 2024-11-10T18:15:00,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/03d7ff0b3ba949b99902649cbc9761b7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/03d7ff0b3ba949b99902649cbc9761b7 2024-11-10T18:15:00,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/625c70f518944a47ad342fd59a25a1e6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/625c70f518944a47ad342fd59a25a1e6 2024-11-10T18:15:00,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/c6d071d8e91541d1aee54e8f65fe79fd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/c6d071d8e91541d1aee54e8f65fe79fd 2024-11-10T18:15:00,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/829f285a22c947a29410d665713c74cd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/829f285a22c947a29410d665713c74cd 2024-11-10T18:15:00,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/34de94a6306e4d0b9c0104b791616399 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/34de94a6306e4d0b9c0104b791616399 2024-11-10T18:15:00,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ce456e60ff9945eea7a67b1aa50744a6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ce456e60ff9945eea7a67b1aa50744a6 2024-11-10T18:15:00,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/688249fe0d594d72a3a98d22a5c8d1ce to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/688249fe0d594d72a3a98d22a5c8d1ce 2024-11-10T18:15:00,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/27a0d2e7ccb742de8c4dec32aa560c48 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/27a0d2e7ccb742de8c4dec32aa560c48 2024-11-10T18:15:00,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fa2277b42654c6e80f93b9eba3f12a7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fa2277b42654c6e80f93b9eba3f12a7 2024-11-10T18:15:00,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/003e36f9b6884e24a7e3afd7a2f87943 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/003e36f9b6884e24a7e3afd7a2f87943 2024-11-10T18:15:00,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/84faa428621640d7baea28d0ad27e088 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/84faa428621640d7baea28d0ad27e088 2024-11-10T18:15:00,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2a3afbaa90414170a72fcdc0f825ab24 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2a3afbaa90414170a72fcdc0f825ab24 2024-11-10T18:15:00,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/010092ccdabe4fab99dd682b02e44a4f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/010092ccdabe4fab99dd682b02e44a4f 2024-11-10T18:15:00,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/5ba279c40ba34c42914d1c48abc7b206 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/5ba279c40ba34c42914d1c48abc7b206 2024-11-10T18:15:00,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/3efebc89ddf847a8a4b3cce0d2a501e1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/3efebc89ddf847a8a4b3cce0d2a501e1 2024-11-10T18:15:00,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ae85fdc32b774842b6765d9bcbfa05be to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ae85fdc32b774842b6765d9bcbfa05be 2024-11-10T18:15:00,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/f995bfc476b944569f0298bcd3a93428 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/f995bfc476b944569f0298bcd3a93428 2024-11-10T18:15:00,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/51202aba26614ed9b10dccc80f18d69f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/51202aba26614ed9b10dccc80f18d69f 2024-11-10T18:15:00,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/41b43359936446bf82fb99334c2e11fb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/41b43359936446bf82fb99334c2e11fb 2024-11-10T18:15:00,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fcacf4f425346218d5d46f1fe817366 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/1fcacf4f425346218d5d46f1fe817366 2024-11-10T18:15:00,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/92e1878efc4642f9b823f29ab6a3b581 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/92e1878efc4642f9b823f29ab6a3b581 2024-11-10T18:15:00,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2d5697697a484c55971de57a630bbceb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2d5697697a484c55971de57a630bbceb 2024-11-10T18:15:00,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/d6654f6cfb664cd5a6af8d4820e86932 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/d6654f6cfb664cd5a6af8d4820e86932 2024-11-10T18:15:00,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2f42195380f4416cb200fe9865b5220d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/2f42195380f4416cb200fe9865b5220d 2024-11-10T18:15:00,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/6445f6dcdc1f4ceeba0cc63ce384ce55 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/6445f6dcdc1f4ceeba0cc63ce384ce55 2024-11-10T18:15:00,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7363eb7b4f5046c49a043a8eab3ec637 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7363eb7b4f5046c49a043a8eab3ec637 2024-11-10T18:15:00,223 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/recovered.edits/557.seqid, newMaxSeqId=557, maxSeqId=1 2024-11-10T18:15:00,224 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. 2024-11-10T18:15:00,224 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 8db0c68c9c5ead4c454abb5a20de2443: Waiting for close lock at 1731262497969Running coprocessor pre-close hooks at 1731262497969Disabling compacts and flushes for region at 1731262497969Disabling writes for close at 1731262498898 (+929 ms)Obtaining lock to block concurrent updates at 1731262498898Preparing flush snapshotting stores in 8db0c68c9c5ead4c454abb5a20de2443 at 1731262498898Finished memstore snapshotting TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731262498899 (+1 ms)Flushing stores of TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443. at 1731262498899Flushing 8db0c68c9c5ead4c454abb5a20de2443/A: creating writer at 1731262498899Flushing 8db0c68c9c5ead4c454abb5a20de2443/A: appending metadata at 1731262498901 (+2 ms)Flushing 8db0c68c9c5ead4c454abb5a20de2443/A: closing flushed file at 1731262498901Flushing 8db0c68c9c5ead4c454abb5a20de2443/B: creating writer at 1731262499309 (+408 ms)Flushing 8db0c68c9c5ead4c454abb5a20de2443/B: appending metadata at 1731262499311 (+2 ms)Flushing 8db0c68c9c5ead4c454abb5a20de2443/B: closing flushed file at 1731262499311Flushing 8db0c68c9c5ead4c454abb5a20de2443/C: creating writer at 1731262499718 (+407 ms)Flushing 8db0c68c9c5ead4c454abb5a20de2443/C: appending metadata at 1731262499720 (+2 ms)Flushing 8db0c68c9c5ead4c454abb5a20de2443/C: closing flushed file at 1731262499720Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61e51188: reopening flushed file at 1731262500128 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@694342bd: reopening flushed file at 1731262500131 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55a68a75: reopening flushed file at 1731262500135 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 8db0c68c9c5ead4c454abb5a20de2443 in 1241ms, sequenceid=554, compaction requested=true at 1731262500139 (+4 ms)Writing region close event to WAL at 1731262500220 (+81 ms)Running coprocessor post-close hooks at 1731262500224 (+4 ms)Closed at 1731262500224 2024-11-10T18:15:00,226 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:15:00,226 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=8db0c68c9c5ead4c454abb5a20de2443, regionState=CLOSED 2024-11-10T18:15:00,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:15:00,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-10T18:15:00,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 8db0c68c9c5ead4c454abb5a20de2443, server=691ea56905d2,32909,1731262402364 in 2.4110 sec 2024-11-10T18:15:00,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-10T18:15:00,230 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8db0c68c9c5ead4c454abb5a20de2443, UNASSIGN in 2.4150 sec 2024-11-10T18:15:00,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-10T18:15:00,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4180 sec 2024-11-10T18:15:00,233 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262500233"}]},"ts":"1731262500233"} 2024-11-10T18:15:00,234 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T18:15:00,234 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T18:15:00,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.4270 sec 2024-11-10T18:15:01,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-10T18:15:01,965 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:01,965 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T18:15:01,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:01,967 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:01,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T18:15:01,968 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:01,970 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:15:01,972 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/recovered.edits] 2024-11-10T18:15:01,974 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/be0787594d7346868cd9bf3307af1dea to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/be0787594d7346868cd9bf3307af1dea 2024-11-10T18:15:01,975 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c0aef3790c7644fc830d6e65f0a88513 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/c0aef3790c7644fc830d6e65f0a88513 2024-11-10T18:15:01,976 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f50630105d944a08adfd05397f928690 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/A/f50630105d944a08adfd05397f928690 2024-11-10T18:15:01,978 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e4d5e867664e78a2338c31331d808e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/52e4d5e867664e78a2338c31331d808e 2024-11-10T18:15:01,978 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/57719b21a28b4e32961c227bffefe57f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/57719b21a28b4e32961c227bffefe57f 2024-11-10T18:15:01,979 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8983a49d667c489fa8c8d2919613ed8a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/B/8983a49d667c489fa8c8d2919613ed8a 2024-11-10T18:15:01,981 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/29a1eeefc94f4cd2a5977d239e67cc79 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/29a1eeefc94f4cd2a5977d239e67cc79 2024-11-10T18:15:01,982 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7e8d4e3369e04e17b686f7dddb33a971 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/7e8d4e3369e04e17b686f7dddb33a971 2024-11-10T18:15:01,983 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ad51b4f653724f848598a8402408ab48 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/C/ad51b4f653724f848598a8402408ab48 2024-11-10T18:15:01,985 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/recovered.edits/557.seqid to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443/recovered.edits/557.seqid 2024-11-10T18:15:01,985 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/8db0c68c9c5ead4c454abb5a20de2443 2024-11-10T18:15:01,985 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T18:15:01,987 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:01,989 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T18:15:01,991 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T18:15:01,992 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:01,992 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T18:15:01,992 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731262501992"}]},"ts":"9223372036854775807"} 2024-11-10T18:15:01,994 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T18:15:01,994 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8db0c68c9c5ead4c454abb5a20de2443, NAME => 'TestAcidGuarantees,,1731262472174.8db0c68c9c5ead4c454abb5a20de2443.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T18:15:01,994 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T18:15:01,994 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731262501994"}]},"ts":"9223372036854775807"} 2024-11-10T18:15:01,995 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T18:15:01,996 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:01,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 31 msec 2024-11-10T18:15:02,018 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T18:15:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-10T18:15:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T18:15:02,075 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:02,086 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=249 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_316223114_22 at /127.0.0.1:55978 [Waiting for operation #264] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1086771773_22 at /127.0.0.1:55754 [Waiting for operation #334] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=322 (was 333), ProcessCount=11 (was 11), AvailableMemoryMB=8043 (was 8109) 2024-11-10T18:15:02,094 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=322, ProcessCount=11, AvailableMemoryMB=8043 2024-11-10T18:15:02,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:15:02,096 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:15:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:02,098 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T18:15:02,098 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:02,098 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-10T18:15:02,099 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T18:15:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T18:15:02,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742157_1333 (size=1061) 2024-11-10T18:15:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T18:15:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T18:15:02,505 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:15:02,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742158_1334 (size=53) 2024-11-10T18:15:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T18:15:02,911 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:15:02,911 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c501ba03299c71686a077099ff2aacf7, disabling compactions & flushes 2024-11-10T18:15:02,911 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:02,911 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:02,911 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. after waiting 0 ms 2024-11-10T18:15:02,911 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:02,911 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:02,911 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c501ba03299c71686a077099ff2aacf7: Waiting for close lock at 1731262502911Disabling compacts and flushes for region at 1731262502911Disabling writes for close at 1731262502911Writing region close event to WAL at 1731262502911Closed at 1731262502911 2024-11-10T18:15:02,912 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T18:15:02,912 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731262502912"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731262502912"}]},"ts":"1731262502912"} 2024-11-10T18:15:02,914 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T18:15:02,915 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T18:15:02,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262502915"}]},"ts":"1731262502915"} 2024-11-10T18:15:02,916 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T18:15:02,916 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, ASSIGN}] 2024-11-10T18:15:02,917 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, ASSIGN 2024-11-10T18:15:02,918 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, ASSIGN; state=OFFLINE, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=false 2024-11-10T18:15:03,068 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:03,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, ASSIGN because future has completed 2024-11-10T18:15:03,070 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:15:03,225 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:03,225 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:15:03,226 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,226 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:15:03,226 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,226 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,227 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,228 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:03,228 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c501ba03299c71686a077099ff2aacf7 columnFamilyName A 2024-11-10T18:15:03,228 DEBUG [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:03,229 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(327): Store=c501ba03299c71686a077099ff2aacf7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:03,229 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,230 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:03,230 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c501ba03299c71686a077099ff2aacf7 columnFamilyName B 2024-11-10T18:15:03,230 DEBUG [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:03,230 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(327): Store=c501ba03299c71686a077099ff2aacf7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:03,230 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,231 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:03,231 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c501ba03299c71686a077099ff2aacf7 columnFamilyName C 2024-11-10T18:15:03,231 DEBUG [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:03,231 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(327): Store=c501ba03299c71686a077099ff2aacf7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:03,231 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:03,232 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,232 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,232 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,233 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,233 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,234 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:15:03,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T18:15:03,235 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,237 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:15:03,237 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened c501ba03299c71686a077099ff2aacf7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62526456, jitterRate=-0.06828320026397705}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:15:03,237 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:03,238 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for c501ba03299c71686a077099ff2aacf7: Running coprocessor pre-open hook at 1731262503226Writing region info on filesystem at 1731262503226Initializing all the Stores at 1731262503227 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262503227Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262503227Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262503227Cleaning up temporary data from old regions at 1731262503233 (+6 ms)Running coprocessor post-open hooks at 1731262503237 (+4 ms)Region opened successfully at 1731262503238 (+1 ms) 2024-11-10T18:15:03,239 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., pid=89, masterSystemTime=1731262503222 2024-11-10T18:15:03,241 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:03,241 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:03,241 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:03,242 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:15:03,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-10T18:15:03,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 in 173 msec 2024-11-10T18:15:03,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-10T18:15:03,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, ASSIGN in 328 msec 2024-11-10T18:15:03,247 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T18:15:03,247 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262503247"}]},"ts":"1731262503247"} 2024-11-10T18:15:03,248 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T18:15:03,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T18:15:03,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1520 sec 2024-11-10T18:15:04,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-10T18:15:04,245 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:04,245 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:04,246 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:04,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:04,247 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c4a4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:04,247 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:04,248 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633d91ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:04,249 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:04,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:04,251 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:15:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:15:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5ab9244e 2024-11-10T18:15:04,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:15:04,252 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:15:04,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:15:04,254 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:15:04,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:04,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742159_1335 (size=1097) 2024-11-10T18:15:04,665 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T18:15:04,665 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T18:15:04,667 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:15:04,669 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, REOPEN/MOVE}] 2024-11-10T18:15:04,669 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, REOPEN/MOVE 2024-11-10T18:15:04,670 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:04,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, REOPEN/MOVE because future has completed 2024-11-10T18:15:04,671 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:15:04,671 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:15:04,823 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:04,823 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:15:04,823 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing c501ba03299c71686a077099ff2aacf7, disabling compactions & flushes 2024-11-10T18:15:04,823 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:04,824 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:04,824 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. after waiting 0 ms 2024-11-10T18:15:04,824 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:04,827 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T18:15:04,827 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:04,827 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for c501ba03299c71686a077099ff2aacf7: Waiting for close lock at 1731262504823Running coprocessor pre-close hooks at 1731262504823Disabling compacts and flushes for region at 1731262504823Disabling writes for close at 1731262504824 (+1 ms)Writing region close event to WAL at 1731262504824Running coprocessor post-close hooks at 1731262504827 (+3 ms)Closed at 1731262504827 2024-11-10T18:15:04,828 WARN [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: c501ba03299c71686a077099ff2aacf7 to self. 2024-11-10T18:15:04,829 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:04,830 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=CLOSED 2024-11-10T18:15:04,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:15:04,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-10T18:15:04,833 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 in 161 msec 2024-11-10T18:15:04,834 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, REOPEN/MOVE; state=CLOSED, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=true 2024-11-10T18:15:04,985 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:04,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, REOPEN/MOVE because future has completed 2024-11-10T18:15:04,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:15:05,141 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,141 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:15:05,141 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,141 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:15:05,141 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,141 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,142 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,143 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:05,143 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c501ba03299c71686a077099ff2aacf7 columnFamilyName A 2024-11-10T18:15:05,144 DEBUG [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:05,145 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(327): Store=c501ba03299c71686a077099ff2aacf7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:05,145 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,145 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:05,145 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c501ba03299c71686a077099ff2aacf7 columnFamilyName B 2024-11-10T18:15:05,146 DEBUG [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:05,146 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(327): Store=c501ba03299c71686a077099ff2aacf7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:05,146 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,146 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:05,146 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c501ba03299c71686a077099ff2aacf7 columnFamilyName C 2024-11-10T18:15:05,147 DEBUG [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:05,147 INFO [StoreOpener-c501ba03299c71686a077099ff2aacf7-1 {}] regionserver.HStore(327): Store=c501ba03299c71686a077099ff2aacf7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:05,147 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,147 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,148 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,148 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,149 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,149 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,149 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:15:05,150 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,151 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened c501ba03299c71686a077099ff2aacf7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59150461, jitterRate=-0.11858944594860077}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:15:05,151 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,151 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for c501ba03299c71686a077099ff2aacf7: Running coprocessor pre-open hook at 1731262505141Writing region info on filesystem at 1731262505141Initializing all the Stores at 1731262505142 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262505142Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262505142Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262505142Cleaning up temporary data from old regions at 1731262505149 (+7 ms)Running coprocessor post-open hooks at 1731262505151 (+2 ms)Region opened successfully at 1731262505151 2024-11-10T18:15:05,152 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., pid=94, masterSystemTime=1731262505138 2024-11-10T18:15:05,154 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,154 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,154 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=OPEN, openSeqNum=5, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,155 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:15:05,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-10T18:15:05,157 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 in 170 msec 2024-11-10T18:15:05,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-10T18:15:05,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, REOPEN/MOVE in 488 msec 2024-11-10T18:15:05,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-10T18:15:05,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 492 msec 2024-11-10T18:15:05,162 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-10T18:15:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-10T18:15:05,163 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:05,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,163 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,164 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,164 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b35f1a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,165 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,166 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffd60a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,167 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,167 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,167 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,167 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bf52d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,169 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41948, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3609ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,170 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,170 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,170 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25fbfe67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,171 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,171 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,172 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c8b1ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,172 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,173 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b4ddb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,174 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41966, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd2957c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,175 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,175 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,175 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,175 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a05af7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,177 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@336431b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,177 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,177 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,177 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,178 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50df2b48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,178 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,179 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58114788, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,180 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,180 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,181 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa6ca21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,181 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,182 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e8c36ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,182 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,182 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,182 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,183 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0cc44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,184 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,184 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,185 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@112f001f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,185 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,186 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ca130c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,187 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183c9114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:05,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:05,188 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca00492, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:05,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:05,190 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:05,190 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d657f67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:05,190 DEBUG [Thread-1550 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,190 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,190 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,191 DEBUG [Thread-1552 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,191 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,191 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,191 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,192 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,192 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,192 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39896, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,193 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,193 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:05,193 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:05,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39922, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:05,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-10T18:15:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,198 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:05,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T18:15:05,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,199 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:05,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:05,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:15:05,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:05,209 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:05,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ac05cf41fd284af79e9f383f7ad5bf4d_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262505206/Put/seqid=0 2024-11-10T18:15:05,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262565245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742160_1336 (size=12154) 2024-11-10T18:15:05,254 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:05,258 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ac05cf41fd284af79e9f383f7ad5bf4d_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ac05cf41fd284af79e9f383f7ad5bf4d_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262565246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,259 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/a7886763e3904f5b8d7560cf8c085713, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/a7886763e3904f5b8d7560cf8c085713 is 175, key is test_row_0/A:col10/1731262505206/Put/seqid=0 2024-11-10T18:15:05,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742161_1337 (size=30955) 2024-11-10T18:15:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262565249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/a7886763e3904f5b8d7560cf8c085713 2024-11-10T18:15:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262565249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262565251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/fd4eda799c604431b06551af3d3a9c73 is 50, key is test_row_0/B:col10/1731262505206/Put/seqid=0 2024-11-10T18:15:05,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742162_1338 (size=12001) 2024-11-10T18:15:05,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/fd4eda799c604431b06551af3d3a9c73 2024-11-10T18:15:05,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T18:15:05,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/f4556d981ec441caa6a880a156b3434a is 50, key is test_row_0/C:col10/1731262505206/Put/seqid=0 2024-11-10T18:15:05,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742163_1339 (size=12001) 2024-11-10T18:15:05,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T18:15:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262565354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,360 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262565364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262565375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262565375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,385 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262565375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,386 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T18:15:05,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T18:15:05,574 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262565565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,575 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262565575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262565594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262565594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262565594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T18:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:05,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/f4556d981ec441caa6a880a156b3434a 2024-11-10T18:15:05,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/a7886763e3904f5b8d7560cf8c085713 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713 2024-11-10T18:15:05,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713, entries=150, sequenceid=16, filesize=30.2 K 2024-11-10T18:15:05,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/fd4eda799c604431b06551af3d3a9c73 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fd4eda799c604431b06551af3d3a9c73 2024-11-10T18:15:05,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fd4eda799c604431b06551af3d3a9c73, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T18:15:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/f4556d981ec441caa6a880a156b3434a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f4556d981ec441caa6a880a156b3434a 2024-11-10T18:15:05,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f4556d981ec441caa6a880a156b3434a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-10T18:15:05,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c501ba03299c71686a077099ff2aacf7 in 531ms, sequenceid=16, compaction requested=false 2024-11-10T18:15:05,737 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T18:15:05,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:05,754 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:15:05,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-10T18:15:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:05,812 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:15:05,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:05,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:05,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dbda459cfca84eeaa5aa12eb2c283d3c_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262505247/Put/seqid=0 2024-11-10T18:15:05,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742164_1340 (size=12154) 2024-11-10T18:15:05,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T18:15:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:05,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:05,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262565907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262565913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262565914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262565914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:05,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:05,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262565915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:05,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:05,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262566024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262566025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262566025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,030 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262566035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262566035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:06,228 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dbda459cfca84eeaa5aa12eb2c283d3c_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dbda459cfca84eeaa5aa12eb2c283d3c_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/cba88bf5c0fd416691136bd8eca88bb3, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/cba88bf5c0fd416691136bd8eca88bb3 is 175, key is test_row_0/A:col10/1731262505247/Put/seqid=0 2024-11-10T18:15:06,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742165_1341 (size=30955) 2024-11-10T18:15:06,236 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/cba88bf5c0fd416691136bd8eca88bb3 2024-11-10T18:15:06,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/dfceb158d28d4185a0163ce826b0a0d2 is 50, key is test_row_0/B:col10/1731262505247/Put/seqid=0 2024-11-10T18:15:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262566235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262566235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262566235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742166_1342 (size=12001) 2024-11-10T18:15:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262566254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262566254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T18:15:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262566555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,561 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262566555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262566555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262566564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262566564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:06,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:06,650 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/dfceb158d28d4185a0163ce826b0a0d2 2024-11-10T18:15:06,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/7f5ec4d27cd4462ca6697aafd36571a4 is 50, key is test_row_0/C:col10/1731262505247/Put/seqid=0 2024-11-10T18:15:06,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742167_1343 (size=12001) 2024-11-10T18:15:07,063 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/7f5ec4d27cd4462ca6697aafd36571a4 2024-11-10T18:15:07,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/cba88bf5c0fd416691136bd8eca88bb3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3 2024-11-10T18:15:07,071 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3, entries=150, sequenceid=36, filesize=30.2 K 2024-11-10T18:15:07,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/dfceb158d28d4185a0163ce826b0a0d2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/dfceb158d28d4185a0163ce826b0a0d2 2024-11-10T18:15:07,075 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/dfceb158d28d4185a0163ce826b0a0d2, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T18:15:07,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/7f5ec4d27cd4462ca6697aafd36571a4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/7f5ec4d27cd4462ca6697aafd36571a4 2024-11-10T18:15:07,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262567065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,080 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/7f5ec4d27cd4462ca6697aafd36571a4, entries=150, sequenceid=36, filesize=11.7 K 2024-11-10T18:15:07,080 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c501ba03299c71686a077099ff2aacf7 in 1268ms, sequenceid=36, compaction requested=false 2024-11-10T18:15:07,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-10T18:15:07,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-10T18:15:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:07,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:15:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:07,084 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:07,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-10T18:15:07,084 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-11-10T18:15:07,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.8890 sec 2024-11-10T18:15:07,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111035c923b46da74ed3b69f091611a45504_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:07,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742168_1344 (size=14594) 2024-11-10T18:15:07,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262567123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262567126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,138 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262567128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262567134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262567244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262567244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262567255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262567255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-10T18:15:07,345 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:07,345 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-10T18:15:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:07,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:07,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:07,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262567465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262567465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262567465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262567474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,502 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:07,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111035c923b46da74ed3b69f091611a45504_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111035c923b46da74ed3b69f091611a45504_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:07,507 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f971a0edac554057a638758d88584d20, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:07,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f971a0edac554057a638758d88584d20 is 175, key is test_row_0/A:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:07,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742169_1345 (size=39549) 2024-11-10T18:15:07,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262567784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262567785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262567785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262567785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:07,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,912 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f971a0edac554057a638758d88584d20 2024-11-10T18:15:07,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/77886a1346994ca7b5d93bf512b650b4 is 50, key is test_row_0/B:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:07,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742170_1346 (size=12001) 2024-11-10T18:15:07,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:07,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:07,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:08,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:08,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262568084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:08,093 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:08,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:08,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262568295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,304 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262568305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262568305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:08,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262568305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:08,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:08,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/77886a1346994ca7b5d93bf512b650b4 2024-11-10T18:15:08,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ea086a9cf4d84f76bbd7e35e7fe4002d is 50, key is test_row_0/C:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:08,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742171_1347 (size=12001) 2024-11-10T18:15:08,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:08,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:08,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ea086a9cf4d84f76bbd7e35e7fe4002d 2024-11-10T18:15:08,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f971a0edac554057a638758d88584d20 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20 2024-11-10T18:15:08,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20, entries=200, sequenceid=49, filesize=38.6 K 2024-11-10T18:15:08,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/77886a1346994ca7b5d93bf512b650b4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/77886a1346994ca7b5d93bf512b650b4 2024-11-10T18:15:08,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/77886a1346994ca7b5d93bf512b650b4, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T18:15:08,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ea086a9cf4d84f76bbd7e35e7fe4002d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ea086a9cf4d84f76bbd7e35e7fe4002d 2024-11-10T18:15:08,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ea086a9cf4d84f76bbd7e35e7fe4002d, entries=150, sequenceid=49, filesize=11.7 K 2024-11-10T18:15:08,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for c501ba03299c71686a077099ff2aacf7 in 1672ms, sequenceid=49, compaction requested=true 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:08,755 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:08,755 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:08,755 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:08,756 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:08,756 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:08,756 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/A is initiating minor compaction (all files) 2024-11-10T18:15:08,756 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/B is initiating minor compaction (all files) 2024-11-10T18:15:08,756 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/B in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,756 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/A in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,757 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fd4eda799c604431b06551af3d3a9c73, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/dfceb158d28d4185a0163ce826b0a0d2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/77886a1346994ca7b5d93bf512b650b4] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.2 K 2024-11-10T18:15:08,757 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=99.1 K 2024-11-10T18:15:08,757 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,757 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20] 2024-11-10T18:15:08,757 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting fd4eda799c604431b06551af3d3a9c73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731262505204 2024-11-10T18:15:08,757 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7886763e3904f5b8d7560cf8c085713, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731262505204 2024-11-10T18:15:08,758 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting dfceb158d28d4185a0163ce826b0a0d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731262505210 2024-11-10T18:15:08,758 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba88bf5c0fd416691136bd8eca88bb3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731262505210 2024-11-10T18:15:08,758 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 77886a1346994ca7b5d93bf512b650b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262505905 2024-11-10T18:15:08,758 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f971a0edac554057a638758d88584d20, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262505905 2024-11-10T18:15:08,767 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#B#compaction#297 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:08,767 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c75f6f565cb943c19187a6d257ae609b is 50, key is test_row_0/B:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:08,773 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:08,785 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c8504e791b974aaeb8c2b772d125f79c_c501ba03299c71686a077099ff2aacf7 store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:08,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742172_1348 (size=12104) 2024-11-10T18:15:08,787 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c8504e791b974aaeb8c2b772d125f79c_c501ba03299c71686a077099ff2aacf7, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:08,788 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c8504e791b974aaeb8c2b772d125f79c_c501ba03299c71686a077099ff2aacf7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:08,792 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c75f6f565cb943c19187a6d257ae609b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c75f6f565cb943c19187a6d257ae609b 2024-11-10T18:15:08,798 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/B of c501ba03299c71686a077099ff2aacf7 into c75f6f565cb943c19187a6d257ae609b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:08,798 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:08,798 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/B, priority=13, startTime=1731262508755; duration=0sec 2024-11-10T18:15:08,798 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:08,798 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:B 2024-11-10T18:15:08,798 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:08,799 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:08,799 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/C is initiating minor compaction (all files) 2024-11-10T18:15:08,799 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/C in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,799 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f4556d981ec441caa6a880a156b3434a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/7f5ec4d27cd4462ca6697aafd36571a4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ea086a9cf4d84f76bbd7e35e7fe4002d] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.2 K 2024-11-10T18:15:08,800 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f4556d981ec441caa6a880a156b3434a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731262505204 2024-11-10T18:15:08,800 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f5ec4d27cd4462ca6697aafd36571a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731262505210 2024-11-10T18:15:08,803 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ea086a9cf4d84f76bbd7e35e7fe4002d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262505905 2024-11-10T18:15:08,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742173_1349 (size=4469) 2024-11-10T18:15:08,806 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#A#compaction#298 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:08,806 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/0c2de2dcef0b4132a8affd8588dc0b16 is 175, key is test_row_0/A:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:08,819 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:08,819 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ed8136193225443fa5b0a28d824252be is 50, key is test_row_0/C:col10/1731262507082/Put/seqid=0 2024-11-10T18:15:08,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742174_1350 (size=31058) 2024-11-10T18:15:08,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742175_1351 (size=12104) 2024-11-10T18:15:08,830 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/0c2de2dcef0b4132a8affd8588dc0b16 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0c2de2dcef0b4132a8affd8588dc0b16 2024-11-10T18:15:08,834 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/A of c501ba03299c71686a077099ff2aacf7 into 0c2de2dcef0b4132a8affd8588dc0b16(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:08,834 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:08,834 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/A, priority=13, startTime=1731262508755; duration=0sec 2024-11-10T18:15:08,834 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:08,834 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:A 2024-11-10T18:15:08,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:08,878 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:08,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:08,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e66901c6933349f3b3645e4b1a558eb2_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262507126/Put/seqid=0 2024-11-10T18:15:08,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742176_1352 (size=12154) 2024-11-10T18:15:09,231 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ed8136193225443fa5b0a28d824252be as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ed8136193225443fa5b0a28d824252be 2024-11-10T18:15:09,236 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/C of c501ba03299c71686a077099ff2aacf7 into ed8136193225443fa5b0a28d824252be(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:09,236 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:09,236 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/C, priority=13, startTime=1731262508755; duration=0sec 2024-11-10T18:15:09,236 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:09,236 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:C 2024-11-10T18:15:09,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:09,295 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110e66901c6933349f3b3645e4b1a558eb2_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e66901c6933349f3b3645e4b1a558eb2_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:09,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f7c4fcfc7dc44de1b1220d9d4243eac1, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:09,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f7c4fcfc7dc44de1b1220d9d4243eac1 is 175, key is test_row_0/A:col10/1731262507126/Put/seqid=0 2024-11-10T18:15:09,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742177_1353 (size=30955) 2024-11-10T18:15:09,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:09,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:09,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262569335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,345 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262569337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,346 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262569344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262569345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,349 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262569455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262569455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262569455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262569455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:09,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262569674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262569674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262569674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262569675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,700 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f7c4fcfc7dc44de1b1220d9d4243eac1 2024-11-10T18:15:09,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/7ec55bb393054e6c8af8debfedc23722 is 50, key is test_row_0/B:col10/1731262507126/Put/seqid=0 2024-11-10T18:15:09,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742178_1354 (size=12001) 2024-11-10T18:15:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262569985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262569985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262569985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262569985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:09,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:10,112 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/7ec55bb393054e6c8af8debfedc23722 2024-11-10T18:15:10,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262570114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:10,118 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,119 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/372a2264dd1f49d7aadc10f5fedbe0a4 is 50, key is test_row_0/C:col10/1731262507126/Put/seqid=0 2024-11-10T18:15:10,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742179_1355 (size=12001) 2024-11-10T18:15:10,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:10,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262570495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:10,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:10,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262570504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:10,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262570504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:10,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262570504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:10,514 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:10,524 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/372a2264dd1f49d7aadc10f5fedbe0a4 2024-11-10T18:15:10,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f7c4fcfc7dc44de1b1220d9d4243eac1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1 2024-11-10T18:15:10,532 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1, entries=150, sequenceid=70, filesize=30.2 K 2024-11-10T18:15:10,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/7ec55bb393054e6c8af8debfedc23722 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/7ec55bb393054e6c8af8debfedc23722 2024-11-10T18:15:10,536 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/7ec55bb393054e6c8af8debfedc23722, entries=150, sequenceid=70, filesize=11.7 K 2024-11-10T18:15:10,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/372a2264dd1f49d7aadc10f5fedbe0a4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/372a2264dd1f49d7aadc10f5fedbe0a4 2024-11-10T18:15:10,541 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/372a2264dd1f49d7aadc10f5fedbe0a4, entries=150, sequenceid=70, filesize=11.7 K 2024-11-10T18:15:10,542 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c501ba03299c71686a077099ff2aacf7 in 1664ms, sequenceid=70, compaction requested=false 2024-11-10T18:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-10T18:15:10,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-10T18:15:10,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-10T18:15:10,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1950 sec 2024-11-10T18:15:10,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.2010 sec 2024-11-10T18:15:10,643 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:15:11,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-10T18:15:11,505 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:11,505 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-10T18:15:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T18:15:11,508 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:11,509 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:11,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:11,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:11,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T18:15:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:11,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:11,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107034a4cbab7b43d694ec62497b1720cd_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:11,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742180_1356 (size=14594) 2024-11-10T18:15:11,523 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:11,528 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411107034a4cbab7b43d694ec62497b1720cd_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107034a4cbab7b43d694ec62497b1720cd_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:11,529 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/59772d86f07c4f70a31d8d6b2f85fa38, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:11,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/59772d86f07c4f70a31d8d6b2f85fa38 is 175, key is test_row_0/A:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:11,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742181_1357 (size=39549) 2024-11-10T18:15:11,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262571557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262571559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262571560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262571565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T18:15:11,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T18:15:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,662 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262571675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262571675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262571675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262571684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T18:15:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T18:15:11,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262571884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262571894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262571894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262571905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:11,935 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/59772d86f07c4f70a31d8d6b2f85fa38 2024-11-10T18:15:11,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/f4bcc1d53438451f8ecf442e6eb453ee is 50, key is test_row_0/B:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:11,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742182_1358 (size=12001) 2024-11-10T18:15:11,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/f4bcc1d53438451f8ecf442e6eb453ee 2024-11-10T18:15:11,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/0f541b2efb6440f18b66c2aaf7520866 is 50, key is test_row_0/C:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:11,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742183_1359 (size=12001) 2024-11-10T18:15:11,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/0f541b2efb6440f18b66c2aaf7520866 2024-11-10T18:15:11,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/59772d86f07c4f70a31d8d6b2f85fa38 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38 2024-11-10T18:15:11,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38, entries=200, sequenceid=85, filesize=38.6 K 2024-11-10T18:15:11,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/f4bcc1d53438451f8ecf442e6eb453ee as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/f4bcc1d53438451f8ecf442e6eb453ee 2024-11-10T18:15:11,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/f4bcc1d53438451f8ecf442e6eb453ee, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T18:15:11,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T18:15:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/0f541b2efb6440f18b66c2aaf7520866 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0f541b2efb6440f18b66c2aaf7520866 2024-11-10T18:15:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:11,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0f541b2efb6440f18b66c2aaf7520866, entries=150, sequenceid=85, filesize=11.7 K 2024-11-10T18:15:11,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c501ba03299c71686a077099ff2aacf7 in 466ms, sequenceid=85, compaction requested=true 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:11,976 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:11,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:11,976 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:11,977 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:11,977 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:11,977 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/A is initiating minor compaction (all files) 2024-11-10T18:15:11,977 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/B is initiating minor compaction (all files) 2024-11-10T18:15:11,977 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/B in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,977 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/A in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,978 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c75f6f565cb943c19187a6d257ae609b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/7ec55bb393054e6c8af8debfedc23722, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/f4bcc1d53438451f8ecf442e6eb453ee] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.3 K 2024-11-10T18:15:11,978 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0c2de2dcef0b4132a8affd8588dc0b16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=99.2 K 2024-11-10T18:15:11,978 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:11,978 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0c2de2dcef0b4132a8affd8588dc0b16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38] 2024-11-10T18:15:11,978 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c75f6f565cb943c19187a6d257ae609b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262505905 2024-11-10T18:15:11,978 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2de2dcef0b4132a8affd8588dc0b16, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262505905 2024-11-10T18:15:11,978 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ec55bb393054e6c8af8debfedc23722, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731262507125 2024-11-10T18:15:11,978 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f4bcc1d53438451f8ecf442e6eb453ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731262509331 2024-11-10T18:15:11,978 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7c4fcfc7dc44de1b1220d9d4243eac1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731262507125 2024-11-10T18:15:11,979 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59772d86f07c4f70a31d8d6b2f85fa38, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731262509331 2024-11-10T18:15:11,985 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:11,985 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#B#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:11,986 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/b5887a466e6a45eaaa10ba040040db63 is 50, key is test_row_0/B:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:11,986 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100368ea5534aa4cd39299d362b793f79e_c501ba03299c71686a077099ff2aacf7 store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:11,988 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100368ea5534aa4cd39299d362b793f79e_c501ba03299c71686a077099ff2aacf7, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:11,988 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100368ea5534aa4cd39299d362b793f79e_c501ba03299c71686a077099ff2aacf7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:11,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742184_1360 (size=12207) 2024-11-10T18:15:11,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742185_1361 (size=4469) 2024-11-10T18:15:12,018 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-10T18:15:12,018 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-10T18:15:12,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-10T18:15:12,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:12,122 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T18:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:12,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:12,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a852bae48b3f4474afd9a260a7e7c979_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262511539/Put/seqid=0 2024-11-10T18:15:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T18:15:12,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742186_1362 (size=12154) 2024-11-10T18:15:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:12,141 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a852bae48b3f4474afd9a260a7e7c979_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a852bae48b3f4474afd9a260a7e7c979_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:12,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/d3afda2107f34638aeb3c930129e24d3, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:12,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/d3afda2107f34638aeb3c930129e24d3 is 175, key is test_row_0/A:col10/1731262511539/Put/seqid=0 2024-11-10T18:15:12,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742187_1363 (size=30955) 2024-11-10T18:15:12,147 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/d3afda2107f34638aeb3c930129e24d3 2024-11-10T18:15:12,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/43b4e73cc25142fda2f9d446c7babe11 is 50, key is test_row_0/B:col10/1731262511539/Put/seqid=0 2024-11-10T18:15:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742188_1364 (size=12001) 2024-11-10T18:15:12,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:12,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:12,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262572234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262572235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,248 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262572236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262572239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262572355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262572355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262572355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262572355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,395 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:12,395 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/863cf38aa9414069b436f9c98bdef606 is 175, key is test_row_0/A:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:12,396 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/b5887a466e6a45eaaa10ba040040db63 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/b5887a466e6a45eaaa10ba040040db63 2024-11-10T18:15:12,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742189_1365 (size=31161) 2024-11-10T18:15:12,403 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/863cf38aa9414069b436f9c98bdef606 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/863cf38aa9414069b436f9c98bdef606 2024-11-10T18:15:12,406 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/B of c501ba03299c71686a077099ff2aacf7 into b5887a466e6a45eaaa10ba040040db63(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:12,406 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:12,406 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/B, priority=13, startTime=1731262511976; duration=0sec 2024-11-10T18:15:12,406 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:12,406 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:B 2024-11-10T18:15:12,406 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:12,407 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:12,407 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/C is initiating minor compaction (all files) 2024-11-10T18:15:12,407 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/C in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:12,407 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ed8136193225443fa5b0a28d824252be, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/372a2264dd1f49d7aadc10f5fedbe0a4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0f541b2efb6440f18b66c2aaf7520866] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.3 K 2024-11-10T18:15:12,408 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8136193225443fa5b0a28d824252be, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731262505905 2024-11-10T18:15:12,408 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/A of c501ba03299c71686a077099ff2aacf7 into 863cf38aa9414069b436f9c98bdef606(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:12,408 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:12,408 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/A, priority=13, startTime=1731262511976; duration=0sec 2024-11-10T18:15:12,408 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 372a2264dd1f49d7aadc10f5fedbe0a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731262507125 2024-11-10T18:15:12,408 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:12,408 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:A 2024-11-10T18:15:12,408 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f541b2efb6440f18b66c2aaf7520866, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731262509331 2024-11-10T18:15:12,415 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#C#compaction#310 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:12,415 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/d3959d06d5be46d0bc5edd2da53cabf9 is 50, key is test_row_0/C:col10/1731262509331/Put/seqid=0 2024-11-10T18:15:12,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742190_1366 (size=12207) 2024-11-10T18:15:12,425 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/d3959d06d5be46d0bc5edd2da53cabf9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/d3959d06d5be46d0bc5edd2da53cabf9 2024-11-10T18:15:12,429 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/C of c501ba03299c71686a077099ff2aacf7 into d3959d06d5be46d0bc5edd2da53cabf9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:12,429 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:12,429 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/C, priority=13, startTime=1731262511976; duration=0sec 2024-11-10T18:15:12,429 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:12,429 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:C 2024-11-10T18:15:12,557 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/43b4e73cc25142fda2f9d446c7babe11 2024-11-10T18:15:12,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/1097ca82b21344b287d7a4a05309d600 is 50, key is test_row_0/C:col10/1731262511539/Put/seqid=0 2024-11-10T18:15:12,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742191_1367 (size=12001) 2024-11-10T18:15:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262572574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262572575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262572575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262572575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,584 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T18:15:12,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262572894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262572894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262572895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262572895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:12,903 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:12,968 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/1097ca82b21344b287d7a4a05309d600 2024-11-10T18:15:12,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/d3afda2107f34638aeb3c930129e24d3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3 2024-11-10T18:15:12,976 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3, entries=150, sequenceid=103, filesize=30.2 K 2024-11-10T18:15:12,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/43b4e73cc25142fda2f9d446c7babe11 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/43b4e73cc25142fda2f9d446c7babe11 2024-11-10T18:15:12,981 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/43b4e73cc25142fda2f9d446c7babe11, entries=150, sequenceid=103, filesize=11.7 K 2024-11-10T18:15:12,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/1097ca82b21344b287d7a4a05309d600 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1097ca82b21344b287d7a4a05309d600 2024-11-10T18:15:12,986 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1097ca82b21344b287d7a4a05309d600, entries=150, sequenceid=103, filesize=11.7 K 2024-11-10T18:15:12,987 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c501ba03299c71686a077099ff2aacf7 in 865ms, sequenceid=103, compaction requested=false 2024-11-10T18:15:12,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:12,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:12,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-10T18:15:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-10T18:15:12,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-10T18:15:12,990 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-11-10T18:15:12,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.4850 sec 2024-11-10T18:15:13,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:13,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T18:15:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:13,412 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:13,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110633bf2afaf5e48e4935d3911b835e320_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:13,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742192_1368 (size=17034) 2024-11-10T18:15:13,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262573441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262573451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262573452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262573453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262573555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262573564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262573574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262573575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,584 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-10T18:15:13,655 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:13,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:13,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-10T18:15:13,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:13,657 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:13,657 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:13,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:13,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:13,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262573764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262573775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262573795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:13,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262573795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:13,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:13,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:13,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:13,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:13,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:13,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:13,826 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:13,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110633bf2afaf5e48e4935d3911b835e320_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110633bf2afaf5e48e4935d3911b835e320_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:13,830 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2588394bae94404aaf83c432fae22f4a, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:13,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2588394bae94404aaf83c432fae22f4a is 175, key is test_row_0/A:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:13,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742193_1369 (size=48139) 2024-11-10T18:15:13,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:13,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:13,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:13,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:14,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262574084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,095 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262574095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262574105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:14,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:14,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262574115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,126 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262574154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,162 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,235 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2588394bae94404aaf83c432fae22f4a 2024-11-10T18:15:14,243 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/67f2ef2862e2458fa93b6afab1d94834 is 50, key is test_row_0/B:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:14,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742194_1370 (size=12001) 2024-11-10T18:15:14,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:14,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:14,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262574605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,615 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262574605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262574624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262574634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:14,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:14,648 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/67f2ef2862e2458fa93b6afab1d94834 2024-11-10T18:15:14,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/6865aa74779548699202f09456d693ab is 50, key is test_row_0/C:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:14,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742195_1371 (size=12001) 2024-11-10T18:15:14,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/6865aa74779548699202f09456d693ab 2024-11-10T18:15:14,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2588394bae94404aaf83c432fae22f4a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a 2024-11-10T18:15:14,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a, entries=250, sequenceid=122, filesize=47.0 K 2024-11-10T18:15:14,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/67f2ef2862e2458fa93b6afab1d94834 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/67f2ef2862e2458fa93b6afab1d94834 2024-11-10T18:15:14,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/67f2ef2862e2458fa93b6afab1d94834, entries=150, sequenceid=122, filesize=11.7 K 2024-11-10T18:15:14,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/6865aa74779548699202f09456d693ab as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/6865aa74779548699202f09456d693ab 2024-11-10T18:15:14,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/6865aa74779548699202f09456d693ab, entries=150, sequenceid=122, filesize=11.7 K 2024-11-10T18:15:14,676 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for c501ba03299c71686a077099ff2aacf7 in 1265ms, sequenceid=122, compaction requested=true 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:14,676 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:14,676 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:14,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:14,678 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:14,678 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110255 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:14,678 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/A is initiating minor compaction (all files) 2024-11-10T18:15:14,678 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/B is initiating minor compaction (all files) 2024-11-10T18:15:14,678 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/B in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,678 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/A in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,678 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/863cf38aa9414069b436f9c98bdef606, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=107.7 K 2024-11-10T18:15:14,678 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/b5887a466e6a45eaaa10ba040040db63, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/43b4e73cc25142fda2f9d446c7babe11, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/67f2ef2862e2458fa93b6afab1d94834] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.4 K 2024-11-10T18:15:14,678 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,678 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/863cf38aa9414069b436f9c98bdef606, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a] 2024-11-10T18:15:14,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b5887a466e6a45eaaa10ba040040db63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731262509331 2024-11-10T18:15:14,679 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 863cf38aa9414069b436f9c98bdef606, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731262509331 2024-11-10T18:15:14,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 43b4e73cc25142fda2f9d446c7babe11, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731262511539 2024-11-10T18:15:14,679 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3afda2107f34638aeb3c930129e24d3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731262511539 2024-11-10T18:15:14,679 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2588394bae94404aaf83c432fae22f4a, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731262512226 2024-11-10T18:15:14,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f2ef2862e2458fa93b6afab1d94834, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731262512226 2024-11-10T18:15:14,685 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:14,689 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110572d5fb5d66b4e43a3366b4ada36a25c_c501ba03299c71686a077099ff2aacf7 store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:14,691 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110572d5fb5d66b4e43a3366b4ada36a25c_c501ba03299c71686a077099ff2aacf7, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:14,691 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110572d5fb5d66b4e43a3366b4ada36a25c_c501ba03299c71686a077099ff2aacf7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:14,691 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#B#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:14,691 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/a9822b771c704e89b2036c1faecbf10b is 50, key is test_row_0/B:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:14,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742197_1373 (size=12309) 2024-11-10T18:15:14,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742196_1372 (size=4469) 2024-11-10T18:15:14,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:14,727 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:14,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2e8fbb304c64f478696e1a1b9e1d18a_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262513424/Put/seqid=0 2024-11-10T18:15:14,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742198_1374 (size=12304) 2024-11-10T18:15:14,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:15,107 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#A#compaction#315 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:15,107 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/0a11a93147f14473892093cd20b67508 is 175, key is test_row_0/A:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:15,110 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/a9822b771c704e89b2036c1faecbf10b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/a9822b771c704e89b2036c1faecbf10b 2024-11-10T18:15:15,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742199_1375 (size=31263) 2024-11-10T18:15:15,117 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/B of c501ba03299c71686a077099ff2aacf7 into a9822b771c704e89b2036c1faecbf10b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:15,117 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:15,117 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/B, priority=13, startTime=1731262514676; duration=0sec 2024-11-10T18:15:15,118 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:15,118 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:B 2024-11-10T18:15:15,118 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:15,119 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/0a11a93147f14473892093cd20b67508 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0a11a93147f14473892093cd20b67508 2024-11-10T18:15:15,120 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:15,120 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/C is initiating minor compaction (all files) 2024-11-10T18:15:15,120 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/C in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:15,120 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/d3959d06d5be46d0bc5edd2da53cabf9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1097ca82b21344b287d7a4a05309d600, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/6865aa74779548699202f09456d693ab] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.4 K 2024-11-10T18:15:15,121 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d3959d06d5be46d0bc5edd2da53cabf9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1731262509331 2024-11-10T18:15:15,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1097ca82b21344b287d7a4a05309d600, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731262511539 2024-11-10T18:15:15,122 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6865aa74779548699202f09456d693ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731262512226 2024-11-10T18:15:15,125 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/A of c501ba03299c71686a077099ff2aacf7 into 0a11a93147f14473892093cd20b67508(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:15,125 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:15,125 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/A, priority=13, startTime=1731262514676; duration=0sec 2024-11-10T18:15:15,125 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:15,125 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:A 2024-11-10T18:15:15,134 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#C#compaction#318 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:15,134 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/b21366c5c60349d2a8fcb99032a9ac6e is 50, key is test_row_0/C:col10/1731262513409/Put/seqid=0 2024-11-10T18:15:15,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742200_1376 (size=12309) 2024-11-10T18:15:15,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:15,151 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b2e8fbb304c64f478696e1a1b9e1d18a_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2e8fbb304c64f478696e1a1b9e1d18a_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:15,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/98ce529d71e4460ab2386dfe788fcf1e, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:15,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/98ce529d71e4460ab2386dfe788fcf1e is 175, key is test_row_0/A:col10/1731262513424/Put/seqid=0 2024-11-10T18:15:15,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742201_1377 (size=31105) 2024-11-10T18:15:15,548 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/b21366c5c60349d2a8fcb99032a9ac6e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/b21366c5c60349d2a8fcb99032a9ac6e 2024-11-10T18:15:15,553 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/C of c501ba03299c71686a077099ff2aacf7 into b21366c5c60349d2a8fcb99032a9ac6e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:15,553 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:15,553 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/C, priority=13, startTime=1731262514676; duration=0sec 2024-11-10T18:15:15,553 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:15,553 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:C 2024-11-10T18:15:15,567 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/98ce529d71e4460ab2386dfe788fcf1e 2024-11-10T18:15:15,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c66f47e5538f48bc92e0154e33f3bed6 is 50, key is test_row_0/B:col10/1731262513424/Put/seqid=0 2024-11-10T18:15:15,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742202_1378 (size=12151) 2024-11-10T18:15:15,582 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c66f47e5538f48bc92e0154e33f3bed6 2024-11-10T18:15:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/0778ffa5d9f545d69d47c23a7e19575e is 50, key is test_row_0/C:col10/1731262513424/Put/seqid=0 2024-11-10T18:15:15,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742203_1379 (size=12151) 2024-11-10T18:15:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:15,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:15,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262575654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,666 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,666 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262575660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262575663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262575666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,675 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262575774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262575785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262575785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:15,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:15,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262575984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:15,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:15,997 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/0778ffa5d9f545d69d47c23a7e19575e 2024-11-10T18:15:16,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/98ce529d71e4460ab2386dfe788fcf1e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e 2024-11-10T18:15:16,005 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e, entries=150, sequenceid=139, filesize=30.4 K 2024-11-10T18:15:16,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c66f47e5538f48bc92e0154e33f3bed6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c66f47e5538f48bc92e0154e33f3bed6 2024-11-10T18:15:16,009 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c66f47e5538f48bc92e0154e33f3bed6, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T18:15:16,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/0778ffa5d9f545d69d47c23a7e19575e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0778ffa5d9f545d69d47c23a7e19575e 2024-11-10T18:15:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262576005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262576005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,013 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0778ffa5d9f545d69d47c23a7e19575e, entries=150, sequenceid=139, filesize=11.9 K 2024-11-10T18:15:16,014 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c501ba03299c71686a077099ff2aacf7 in 1287ms, sequenceid=139, compaction requested=false 2024-11-10T18:15:16,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:16,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:16,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-10T18:15:16,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-10T18:15:16,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-10T18:15:16,016 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3570 sec 2024-11-10T18:15:16,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.3620 sec 2024-11-10T18:15:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:16,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-10T18:15:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:16,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:16,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105f414db5d1414d30b72c3f3148319be5_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:16,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742204_1380 (size=14794) 2024-11-10T18:15:16,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262576346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262576351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262576357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262576464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262576465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262576475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262576684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262576685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262576685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:16,712 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:16,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105f414db5d1414d30b72c3f3148319be5_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105f414db5d1414d30b72c3f3148319be5_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:16,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/bf296337132c4700b8149a27c659e204, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:16,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/bf296337132c4700b8149a27c659e204 is 175, key is test_row_0/A:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:16,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742205_1381 (size=39749) 2024-11-10T18:15:16,724 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/bf296337132c4700b8149a27c659e204 2024-11-10T18:15:16,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/75efb5b2d91449629934dbc83b47442b is 50, key is test_row_0/B:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:16,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742206_1382 (size=12151) 2024-11-10T18:15:16,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/75efb5b2d91449629934dbc83b47442b 2024-11-10T18:15:16,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/57b002fdb3b8491a9f8cfe23e7d8fb63 is 50, key is test_row_0/C:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742207_1383 (size=12151) 2024-11-10T18:15:16,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:16,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262576995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:16,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262577004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262577004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/57b002fdb3b8491a9f8cfe23e7d8fb63 2024-11-10T18:15:17,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/bf296337132c4700b8149a27c659e204 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204 2024-11-10T18:15:17,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204, entries=200, sequenceid=158, filesize=38.8 K 2024-11-10T18:15:17,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/75efb5b2d91449629934dbc83b47442b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/75efb5b2d91449629934dbc83b47442b 2024-11-10T18:15:17,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/75efb5b2d91449629934dbc83b47442b, entries=150, sequenceid=158, filesize=11.9 K 2024-11-10T18:15:17,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/57b002fdb3b8491a9f8cfe23e7d8fb63 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/57b002fdb3b8491a9f8cfe23e7d8fb63 2024-11-10T18:15:17,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/57b002fdb3b8491a9f8cfe23e7d8fb63, entries=150, sequenceid=158, filesize=11.9 K 2024-11-10T18:15:17,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for c501ba03299c71686a077099ff2aacf7 in 861ms, sequenceid=158, compaction requested=true 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:17,162 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:17,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:17,162 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/A is initiating minor compaction (all files) 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/B is initiating minor compaction (all files) 2024-11-10T18:15:17,163 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/A in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:17,163 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/B in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:17,163 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0a11a93147f14473892093cd20b67508, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=99.7 K 2024-11-10T18:15:17,163 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/a9822b771c704e89b2036c1faecbf10b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c66f47e5538f48bc92e0154e33f3bed6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/75efb5b2d91449629934dbc83b47442b] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.8 K 2024-11-10T18:15:17,163 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0a11a93147f14473892093cd20b67508, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204] 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a9822b771c704e89b2036c1faecbf10b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731262512226 2024-11-10T18:15:17,163 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a11a93147f14473892093cd20b67508, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731262512226 2024-11-10T18:15:17,164 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c66f47e5538f48bc92e0154e33f3bed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731262513424 2024-11-10T18:15:17,164 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98ce529d71e4460ab2386dfe788fcf1e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731262513424 2024-11-10T18:15:17,164 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 75efb5b2d91449629934dbc83b47442b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731262515650 2024-11-10T18:15:17,164 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf296337132c4700b8149a27c659e204, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731262515646 2024-11-10T18:15:17,172 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:17,174 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#B#compaction#325 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:17,174 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/2db78c1790ad4adab9f7ee821330f90b is 50, key is test_row_0/B:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:17,180 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110b76a324d0147481887e7563a62dd7c35_c501ba03299c71686a077099ff2aacf7 store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:17,181 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110b76a324d0147481887e7563a62dd7c35_c501ba03299c71686a077099ff2aacf7, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:17,182 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b76a324d0147481887e7563a62dd7c35_c501ba03299c71686a077099ff2aacf7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:17,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742208_1384 (size=12561) 2024-11-10T18:15:17,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742209_1385 (size=4469) 2024-11-10T18:15:17,194 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#A#compaction#324 average throughput is 1.11 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:17,195 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/57e4f82ae6e14175a2da222797fab390 is 175, key is test_row_0/A:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:17,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742210_1386 (size=31515) 2024-11-10T18:15:17,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:17,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-10T18:15:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:17,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:17,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b68caaeba39e4f34bb2ae1669a036faa_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262516350/Put/seqid=0 2024-11-10T18:15:17,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742211_1387 (size=14794) 2024-11-10T18:15:17,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262577556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,568 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262577559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262577568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,591 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/2db78c1790ad4adab9f7ee821330f90b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2db78c1790ad4adab9f7ee821330f90b 2024-11-10T18:15:17,594 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/B of c501ba03299c71686a077099ff2aacf7 into 2db78c1790ad4adab9f7ee821330f90b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:17,595 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:17,595 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/B, priority=13, startTime=1731262517162; duration=0sec 2024-11-10T18:15:17,595 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:17,595 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:B 2024-11-10T18:15:17,595 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:17,595 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:17,595 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/C is initiating minor compaction (all files) 2024-11-10T18:15:17,596 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/C in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:17,596 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/b21366c5c60349d2a8fcb99032a9ac6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0778ffa5d9f545d69d47c23a7e19575e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/57b002fdb3b8491a9f8cfe23e7d8fb63] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=35.8 K 2024-11-10T18:15:17,596 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b21366c5c60349d2a8fcb99032a9ac6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731262512226 2024-11-10T18:15:17,597 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0778ffa5d9f545d69d47c23a7e19575e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731262513424 2024-11-10T18:15:17,597 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 57b002fdb3b8491a9f8cfe23e7d8fb63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731262515650 2024-11-10T18:15:17,602 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/57e4f82ae6e14175a2da222797fab390 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/57e4f82ae6e14175a2da222797fab390 2024-11-10T18:15:17,606 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#C#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:17,606 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/A of c501ba03299c71686a077099ff2aacf7 into 57e4f82ae6e14175a2da222797fab390(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:17,606 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:17,606 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/A, priority=13, startTime=1731262517162; duration=0sec 2024-11-10T18:15:17,606 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:17,606 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:A 2024-11-10T18:15:17,606 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/71c30463a3164500ade8cde5b4a6b9e1 is 50, key is test_row_0/C:col10/1731262515664/Put/seqid=0 2024-11-10T18:15:17,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742212_1388 (size=12561) 2024-11-10T18:15:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262577675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262577675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262577684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262577684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,693 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,694 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-10T18:15:17,815 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:17,815 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:17,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-10T18:15:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:17,817 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:17,818 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:17,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:17,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262577885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262577885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:17,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262577904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:17,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:17,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:17,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b68caaeba39e4f34bb2ae1669a036faa_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b68caaeba39e4f34bb2ae1669a036faa_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:17,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/7d94bea873b94895b3bf40246cd1ad21, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:17,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/7d94bea873b94895b3bf40246cd1ad21 is 175, key is test_row_0/A:col10/1731262516350/Put/seqid=0 2024-11-10T18:15:17,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742213_1389 (size=39749) 2024-11-10T18:15:17,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:17,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:17,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:17,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:17,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:17,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:17,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,016 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/71c30463a3164500ade8cde5b4a6b9e1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/71c30463a3164500ade8cde5b4a6b9e1 2024-11-10T18:15:18,021 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/C of c501ba03299c71686a077099ff2aacf7 into 71c30463a3164500ade8cde5b4a6b9e1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:18,022 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:18,022 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/C, priority=13, startTime=1731262517162; duration=0sec 2024-11-10T18:15:18,022 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:18,022 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:C 2024-11-10T18:15:18,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:18,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:18,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:18,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262578195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:18,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262578195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:18,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262578224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:18,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/7d94bea873b94895b3bf40246cd1ad21 2024-11-10T18:15:18,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/fabf635b0b964cdbbd479601ea732125 is 50, key is test_row_0/B:col10/1731262516350/Put/seqid=0 2024-11-10T18:15:18,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742214_1390 (size=12151) 2024-11-10T18:15:18,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:18,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:18,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262578704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:18,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:18,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262578704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:18,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:18,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:18,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262578745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:18,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/fabf635b0b964cdbbd479601ea732125 2024-11-10T18:15:18,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:18,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/01a71bcb5b7146f1bc66c7a2093d5273 is 50, key is test_row_0/C:col10/1731262516350/Put/seqid=0 2024-11-10T18:15:18,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742215_1391 (size=12151) 2024-11-10T18:15:18,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:18,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:18,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:19,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:19,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:19,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:19,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/01a71bcb5b7146f1bc66c7a2093d5273 2024-11-10T18:15:19,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/7d94bea873b94895b3bf40246cd1ad21 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21 2024-11-10T18:15:19,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21, entries=200, sequenceid=175, filesize=38.8 K 2024-11-10T18:15:19,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/fabf635b0b964cdbbd479601ea732125 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fabf635b0b964cdbbd479601ea732125 2024-11-10T18:15:19,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fabf635b0b964cdbbd479601ea732125, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T18:15:19,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/01a71bcb5b7146f1bc66c7a2093d5273 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01a71bcb5b7146f1bc66c7a2093d5273 2024-11-10T18:15:19,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01a71bcb5b7146f1bc66c7a2093d5273, entries=150, sequenceid=175, filesize=11.9 K 2024-11-10T18:15:19,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for c501ba03299c71686a077099ff2aacf7 in 1661ms, sequenceid=175, compaction requested=false 2024-11-10T18:15:19,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:19,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-10T18:15:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:19,194 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T18:15:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:19,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:19,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:19,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:19,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014abd008f49e4d3fb0ff8f65189f211e_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262517567/Put/seqid=0 2024-11-10T18:15:19,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742216_1392 (size=12304) 2024-11-10T18:15:19,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:19,610 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111014abd008f49e4d3fb0ff8f65189f211e_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014abd008f49e4d3fb0ff8f65189f211e_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2d2acbc5599740cdbc5f343364903684, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:19,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2d2acbc5599740cdbc5f343364903684 is 175, key is test_row_0/A:col10/1731262517567/Put/seqid=0 2024-11-10T18:15:19,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742217_1393 (size=31105) 2024-11-10T18:15:19,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:19,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:19,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262579770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:19,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:19,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262579774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:19,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262579775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:19,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262579884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:19,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262579895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:19,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,903 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:19,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262579895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:19,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:19,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:20,015 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2d2acbc5599740cdbc5f343364903684 2024-11-10T18:15:20,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c63bbc0d55b24328864678fca19c0629 is 50, key is test_row_0/B:col10/1731262517567/Put/seqid=0 2024-11-10T18:15:20,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742218_1394 (size=12151) 2024-11-10T18:15:20,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262580104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,114 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262580115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262580115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,125 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,427 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c63bbc0d55b24328864678fca19c0629 2024-11-10T18:15:20,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ca50f5f5915641869865e236df4a400a is 50, key is test_row_0/C:col10/1731262517567/Put/seqid=0 2024-11-10T18:15:20,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262580424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742219_1395 (size=12151) 2024-11-10T18:15:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262580435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262580435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,493 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T18:15:20,838 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ca50f5f5915641869865e236df4a400a 2024-11-10T18:15:20,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/2d2acbc5599740cdbc5f343364903684 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684 2024-11-10T18:15:20,846 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684, entries=150, sequenceid=193, filesize=30.4 K 2024-11-10T18:15:20,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/c63bbc0d55b24328864678fca19c0629 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c63bbc0d55b24328864678fca19c0629 2024-11-10T18:15:20,850 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c63bbc0d55b24328864678fca19c0629, entries=150, sequenceid=193, filesize=11.9 K 2024-11-10T18:15:20,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/ca50f5f5915641869865e236df4a400a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ca50f5f5915641869865e236df4a400a 2024-11-10T18:15:20,854 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ca50f5f5915641869865e236df4a400a, entries=150, sequenceid=193, filesize=11.9 K 2024-11-10T18:15:20,855 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c501ba03299c71686a077099ff2aacf7 in 1661ms, sequenceid=193, compaction requested=true 2024-11-10T18:15:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:20,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-10T18:15:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-10T18:15:20,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-10T18:15:20,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0390 sec 2024-11-10T18:15:20,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 3.0440 sec 2024-11-10T18:15:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:20,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T18:15:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:20,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111018ccc4174b104d26b8d1a3d11bccdd92_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:20,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742220_1396 (size=14794) 2024-11-10T18:15:20,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262580982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262580983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262580997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:20,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262581104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262581104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262581104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262581324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262581324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262581324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:21,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111018ccc4174b104d26b8d1a3d11bccdd92_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111018ccc4174b104d26b8d1a3d11bccdd92_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:21,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/3bf00193603d4a8abad1adb3002fdd09, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:21,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/3bf00193603d4a8abad1adb3002fdd09 is 175, key is test_row_0/A:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:21,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742221_1397 (size=39749) 2024-11-10T18:15:21,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262581645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262581645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262581645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39946 deadline: 1731262581705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:21,716 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:21,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:21,772 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/3bf00193603d4a8abad1adb3002fdd09 2024-11-10T18:15:21,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/0401453021ba454eb879b6aa22a73ed6 is 50, key is test_row_0/B:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742222_1398 (size=12151) 2024-11-10T18:15:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-10T18:15:21,975 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:21,976 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-10T18:15:21,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:21,978 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:21,978 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:21,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:22,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:22,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T18:15:22,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:22,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262582164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:22,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:22,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262582164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:22,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:22,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262582165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:22,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/0401453021ba454eb879b6aa22a73ed6 2024-11-10T18:15:22,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/01275a89f9d247748a72fb728da58da2 is 50, key is test_row_0/C:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:22,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742223_1399 (size=12151) 2024-11-10T18:15:22,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T18:15:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:22,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T18:15:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,437 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,590 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T18:15:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,590 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:22,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/01275a89f9d247748a72fb728da58da2 2024-11-10T18:15:22,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/3bf00193603d4a8abad1adb3002fdd09 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09 2024-11-10T18:15:22,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09, entries=200, sequenceid=211, filesize=38.8 K 2024-11-10T18:15:22,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/0401453021ba454eb879b6aa22a73ed6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/0401453021ba454eb879b6aa22a73ed6 2024-11-10T18:15:22,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:22,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/0401453021ba454eb879b6aa22a73ed6, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T18:15:22,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/01275a89f9d247748a72fb728da58da2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01275a89f9d247748a72fb728da58da2 2024-11-10T18:15:22,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01275a89f9d247748a72fb728da58da2, entries=150, sequenceid=211, filesize=11.9 K 2024-11-10T18:15:22,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c501ba03299c71686a077099ff2aacf7 in 1658ms, sequenceid=211, compaction requested=true 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:22,609 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:22,609 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:22,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:22,610 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:22,611 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/B is initiating minor compaction (all files) 2024-11-10T18:15:22,611 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/B in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,611 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2db78c1790ad4adab9f7ee821330f90b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fabf635b0b964cdbbd479601ea732125, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c63bbc0d55b24328864678fca19c0629, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/0401453021ba454eb879b6aa22a73ed6] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=47.9 K 2024-11-10T18:15:22,611 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142118 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:22,611 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/A is initiating minor compaction (all files) 2024-11-10T18:15:22,611 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/A in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,611 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/57e4f82ae6e14175a2da222797fab390, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=138.8 K 2024-11-10T18:15:22,611 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=57.69 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,611 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/57e4f82ae6e14175a2da222797fab390, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09] 2024-11-10T18:15:22,611 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 2db78c1790ad4adab9f7ee821330f90b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731262515650 2024-11-10T18:15:22,612 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e4f82ae6e14175a2da222797fab390, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731262515650 2024-11-10T18:15:22,612 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting fabf635b0b964cdbbd479601ea732125, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731262516350 2024-11-10T18:15:22,612 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d94bea873b94895b3bf40246cd1ad21, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731262516332 2024-11-10T18:15:22,612 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c63bbc0d55b24328864678fca19c0629, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731262517555 2024-11-10T18:15:22,612 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d2acbc5599740cdbc5f343364903684, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731262517555 2024-11-10T18:15:22,613 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0401453021ba454eb879b6aa22a73ed6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731262519772 2024-11-10T18:15:22,613 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bf00193603d4a8abad1adb3002fdd09, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731262519764 2024-11-10T18:15:22,622 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#B#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:22,622 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/aac4967826234eaa87cbaf377b2592b7 is 50, key is test_row_0/B:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:22,623 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:22,629 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411105f8f46ca394d4bf9936c217108321000_c501ba03299c71686a077099ff2aacf7 store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:22,631 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411105f8f46ca394d4bf9936c217108321000_c501ba03299c71686a077099ff2aacf7, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:22,631 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411105f8f46ca394d4bf9936c217108321000_c501ba03299c71686a077099ff2aacf7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:22,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742224_1400 (size=12697) 2024-11-10T18:15:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742225_1401 (size=4469) 2024-11-10T18:15:22,644 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#A#compaction#337 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:22,645 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/4a24aa525e9e4de5ba4160a6737ca34e is 175, key is test_row_0/A:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:22,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742226_1402 (size=31651) 2024-11-10T18:15:22,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:22,743 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:22,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:22,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2f10615544d4552a0e37a77ff5566b3_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262520995/Put/seqid=0 2024-11-10T18:15:22,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742227_1403 (size=12304) 2024-11-10T18:15:23,041 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/aac4967826234eaa87cbaf377b2592b7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/aac4967826234eaa87cbaf377b2592b7 2024-11-10T18:15:23,046 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c501ba03299c71686a077099ff2aacf7/B of c501ba03299c71686a077099ff2aacf7 into aac4967826234eaa87cbaf377b2592b7(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:23,046 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:23,046 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/B, priority=12, startTime=1731262522609; duration=0sec 2024-11-10T18:15:23,046 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:23,046 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:B 2024-11-10T18:15:23,046 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:23,047 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:23,047 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/C is initiating minor compaction (all files) 2024-11-10T18:15:23,047 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/C in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:23,047 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/71c30463a3164500ade8cde5b4a6b9e1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01a71bcb5b7146f1bc66c7a2093d5273, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ca50f5f5915641869865e236df4a400a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01275a89f9d247748a72fb728da58da2] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=47.9 K 2024-11-10T18:15:23,048 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 71c30463a3164500ade8cde5b4a6b9e1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731262515650 2024-11-10T18:15:23,048 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 01a71bcb5b7146f1bc66c7a2093d5273, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731262516350 2024-11-10T18:15:23,048 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ca50f5f5915641869865e236df4a400a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731262517555 2024-11-10T18:15:23,049 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 01275a89f9d247748a72fb728da58da2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731262519772 2024-11-10T18:15:23,051 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/4a24aa525e9e4de5ba4160a6737ca34e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/4a24aa525e9e4de5ba4160a6737ca34e 2024-11-10T18:15:23,056 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c501ba03299c71686a077099ff2aacf7/A of c501ba03299c71686a077099ff2aacf7 into 4a24aa525e9e4de5ba4160a6737ca34e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:23,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:23,056 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/A, priority=12, startTime=1731262522609; duration=0sec 2024-11-10T18:15:23,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:23,056 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:A 2024-11-10T18:15:23,060 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#C#compaction#339 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 57.69 MB/second 2024-11-10T18:15:23,061 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/efadf3f2cec0447cb520d08af9149309 is 50, key is test_row_0/C:col10/1731262520949/Put/seqid=0 2024-11-10T18:15:23,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742228_1404 (size=12697) 2024-11-10T18:15:23,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:23,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:23,158 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2f10615544d4552a0e37a77ff5566b3_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2f10615544d4552a0e37a77ff5566b3_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/26e600eaabe949268c962853a5e6b5e0, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/26e600eaabe949268c962853a5e6b5e0 is 175, key is test_row_0/A:col10/1731262520995/Put/seqid=0 2024-11-10T18:15:23,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742229_1405 (size=31105) 2024-11-10T18:15:23,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:23,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. as already flushing 2024-11-10T18:15:23,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262583224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262583233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262583234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262583345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262583345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262583354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,468 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/efadf3f2cec0447cb520d08af9149309 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/efadf3f2cec0447cb520d08af9149309 2024-11-10T18:15:23,473 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c501ba03299c71686a077099ff2aacf7/C of c501ba03299c71686a077099ff2aacf7 into efadf3f2cec0447cb520d08af9149309(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:23,473 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:23,473 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/C, priority=12, startTime=1731262522609; duration=0sec 2024-11-10T18:15:23,473 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:23,473 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:C 2024-11-10T18:15:23,563 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/26e600eaabe949268c962853a5e6b5e0 2024-11-10T18:15:23,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/1bcbe3df665d434d952ea86dc7ea4c7c is 50, key is test_row_0/B:col10/1731262520995/Put/seqid=0 2024-11-10T18:15:23,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742230_1406 (size=12151) 2024-11-10T18:15:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262583565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,575 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262583565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262583574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262583884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262583884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:23,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262583895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:23,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:23,975 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/1bcbe3df665d434d952ea86dc7ea4c7c 2024-11-10T18:15:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/9d70c41cbc854c57a1488db5ba6ad3c4 is 50, key is test_row_0/C:col10/1731262520995/Put/seqid=0 2024-11-10T18:15:23,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742231_1407 (size=12151) 2024-11-10T18:15:24,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:24,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39896 deadline: 1731262584185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,195 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in c501ba03299c71686a077099ff2aacf7 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18289 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,196 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,386 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/9d70c41cbc854c57a1488db5ba6ad3c4 2024-11-10T18:15:24,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/26e600eaabe949268c962853a5e6b5e0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0 2024-11-10T18:15:24,393 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0, entries=150, sequenceid=225, filesize=30.4 K 2024-11-10T18:15:24,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/1bcbe3df665d434d952ea86dc7ea4c7c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/1bcbe3df665d434d952ea86dc7ea4c7c 2024-11-10T18:15:24,397 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/1bcbe3df665d434d952ea86dc7ea4c7c, entries=150, sequenceid=225, filesize=11.9 K 2024-11-10T18:15:24,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/9d70c41cbc854c57a1488db5ba6ad3c4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9d70c41cbc854c57a1488db5ba6ad3c4 2024-11-10T18:15:24,401 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9d70c41cbc854c57a1488db5ba6ad3c4, entries=150, sequenceid=225, filesize=11.9 K 2024-11-10T18:15:24,402 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c501ba03299c71686a077099ff2aacf7 in 1658ms, sequenceid=225, compaction requested=false 2024-11-10T18:15:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-10T18:15:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-10T18:15:24,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-10T18:15:24,405 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4250 sec 2024-11-10T18:15:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:24,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T18:15:24,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.4310 sec 2024-11-10T18:15:24,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:24,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:24,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:24,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:24,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:24,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:24,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d818b0aff13d491bab76d35c38fd6b34_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:24,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742232_1408 (size=14794) 2024-11-10T18:15:24,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262584434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262584440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262584440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262584544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262584554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262584554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,564 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262584764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262584774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262584774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:24,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:24,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:24,822 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d818b0aff13d491bab76d35c38fd6b34_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d818b0aff13d491bab76d35c38fd6b34_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:24,822 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/8ebcbdb7aa584d50bfb9a3f89ed380ac, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:24,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/8ebcbdb7aa584d50bfb9a3f89ed380ac is 175, key is test_row_0/A:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:24,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742233_1409 (size=39749) 2024-11-10T18:15:24,963 DEBUG [regionserver/691ea56905d2:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-10T18:15:24,964 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0c2de2dcef0b4132a8affd8588dc0b16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/863cf38aa9414069b436f9c98bdef606, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0a11a93147f14473892093cd20b67508, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/57e4f82ae6e14175a2da222797fab390, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09] to archive 2024-11-10T18:15:24,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:24,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/a7886763e3904f5b8d7560cf8c085713 2024-11-10T18:15:24,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/cba88bf5c0fd416691136bd8eca88bb3 2024-11-10T18:15:24,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f971a0edac554057a638758d88584d20 2024-11-10T18:15:24,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0c2de2dcef0b4132a8affd8588dc0b16 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0c2de2dcef0b4132a8affd8588dc0b16 2024-11-10T18:15:24,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f7c4fcfc7dc44de1b1220d9d4243eac1 2024-11-10T18:15:24,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/59772d86f07c4f70a31d8d6b2f85fa38 2024-11-10T18:15:24,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/863cf38aa9414069b436f9c98bdef606 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/863cf38aa9414069b436f9c98bdef606 2024-11-10T18:15:24,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/d3afda2107f34638aeb3c930129e24d3 2024-11-10T18:15:24,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2588394bae94404aaf83c432fae22f4a 2024-11-10T18:15:24,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0a11a93147f14473892093cd20b67508 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/0a11a93147f14473892093cd20b67508 2024-11-10T18:15:24,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/98ce529d71e4460ab2386dfe788fcf1e 2024-11-10T18:15:24,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/bf296337132c4700b8149a27c659e204 2024-11-10T18:15:24,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/57e4f82ae6e14175a2da222797fab390 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/57e4f82ae6e14175a2da222797fab390 2024-11-10T18:15:24,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/7d94bea873b94895b3bf40246cd1ad21 2024-11-10T18:15:24,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/2d2acbc5599740cdbc5f343364903684 2024-11-10T18:15:24,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/3bf00193603d4a8abad1adb3002fdd09 2024-11-10T18:15:24,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fd4eda799c604431b06551af3d3a9c73, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/dfceb158d28d4185a0163ce826b0a0d2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c75f6f565cb943c19187a6d257ae609b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/77886a1346994ca7b5d93bf512b650b4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/7ec55bb393054e6c8af8debfedc23722, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/b5887a466e6a45eaaa10ba040040db63, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/f4bcc1d53438451f8ecf442e6eb453ee, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/43b4e73cc25142fda2f9d446c7babe11, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/a9822b771c704e89b2036c1faecbf10b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/67f2ef2862e2458fa93b6afab1d94834, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c66f47e5538f48bc92e0154e33f3bed6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2db78c1790ad4adab9f7ee821330f90b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/75efb5b2d91449629934dbc83b47442b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fabf635b0b964cdbbd479601ea732125, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c63bbc0d55b24328864678fca19c0629, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/0401453021ba454eb879b6aa22a73ed6] to archive 2024-11-10T18:15:24,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:24,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fd4eda799c604431b06551af3d3a9c73 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fd4eda799c604431b06551af3d3a9c73 2024-11-10T18:15:24,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/dfceb158d28d4185a0163ce826b0a0d2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/dfceb158d28d4185a0163ce826b0a0d2 2024-11-10T18:15:24,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c75f6f565cb943c19187a6d257ae609b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c75f6f565cb943c19187a6d257ae609b 2024-11-10T18:15:24,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/77886a1346994ca7b5d93bf512b650b4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/77886a1346994ca7b5d93bf512b650b4 2024-11-10T18:15:24,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/7ec55bb393054e6c8af8debfedc23722 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/7ec55bb393054e6c8af8debfedc23722 2024-11-10T18:15:24,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/b5887a466e6a45eaaa10ba040040db63 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/b5887a466e6a45eaaa10ba040040db63 2024-11-10T18:15:24,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/f4bcc1d53438451f8ecf442e6eb453ee to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/f4bcc1d53438451f8ecf442e6eb453ee 2024-11-10T18:15:24,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/43b4e73cc25142fda2f9d446c7babe11 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/43b4e73cc25142fda2f9d446c7babe11 2024-11-10T18:15:24,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/a9822b771c704e89b2036c1faecbf10b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/a9822b771c704e89b2036c1faecbf10b 2024-11-10T18:15:24,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/67f2ef2862e2458fa93b6afab1d94834 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/67f2ef2862e2458fa93b6afab1d94834 2024-11-10T18:15:24,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c66f47e5538f48bc92e0154e33f3bed6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c66f47e5538f48bc92e0154e33f3bed6 2024-11-10T18:15:24,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2db78c1790ad4adab9f7ee821330f90b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2db78c1790ad4adab9f7ee821330f90b 2024-11-10T18:15:24,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/75efb5b2d91449629934dbc83b47442b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/75efb5b2d91449629934dbc83b47442b 2024-11-10T18:15:24,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fabf635b0b964cdbbd479601ea732125 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/fabf635b0b964cdbbd479601ea732125 2024-11-10T18:15:24,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c63bbc0d55b24328864678fca19c0629 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/c63bbc0d55b24328864678fca19c0629 2024-11-10T18:15:25,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/0401453021ba454eb879b6aa22a73ed6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/0401453021ba454eb879b6aa22a73ed6 2024-11-10T18:15:25,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f4556d981ec441caa6a880a156b3434a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/7f5ec4d27cd4462ca6697aafd36571a4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ed8136193225443fa5b0a28d824252be, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ea086a9cf4d84f76bbd7e35e7fe4002d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/372a2264dd1f49d7aadc10f5fedbe0a4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/d3959d06d5be46d0bc5edd2da53cabf9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0f541b2efb6440f18b66c2aaf7520866, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1097ca82b21344b287d7a4a05309d600, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/b21366c5c60349d2a8fcb99032a9ac6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/6865aa74779548699202f09456d693ab, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0778ffa5d9f545d69d47c23a7e19575e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/71c30463a3164500ade8cde5b4a6b9e1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/57b002fdb3b8491a9f8cfe23e7d8fb63, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01a71bcb5b7146f1bc66c7a2093d5273, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ca50f5f5915641869865e236df4a400a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01275a89f9d247748a72fb728da58da2] to archive 2024-11-10T18:15:25,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:25,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f4556d981ec441caa6a880a156b3434a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f4556d981ec441caa6a880a156b3434a 2024-11-10T18:15:25,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/7f5ec4d27cd4462ca6697aafd36571a4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/7f5ec4d27cd4462ca6697aafd36571a4 2024-11-10T18:15:25,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ed8136193225443fa5b0a28d824252be to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ed8136193225443fa5b0a28d824252be 2024-11-10T18:15:25,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ea086a9cf4d84f76bbd7e35e7fe4002d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ea086a9cf4d84f76bbd7e35e7fe4002d 2024-11-10T18:15:25,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/372a2264dd1f49d7aadc10f5fedbe0a4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/372a2264dd1f49d7aadc10f5fedbe0a4 2024-11-10T18:15:25,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/d3959d06d5be46d0bc5edd2da53cabf9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/d3959d06d5be46d0bc5edd2da53cabf9 2024-11-10T18:15:25,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0f541b2efb6440f18b66c2aaf7520866 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0f541b2efb6440f18b66c2aaf7520866 2024-11-10T18:15:25,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1097ca82b21344b287d7a4a05309d600 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1097ca82b21344b287d7a4a05309d600 2024-11-10T18:15:25,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/b21366c5c60349d2a8fcb99032a9ac6e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/b21366c5c60349d2a8fcb99032a9ac6e 2024-11-10T18:15:25,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/6865aa74779548699202f09456d693ab to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/6865aa74779548699202f09456d693ab 2024-11-10T18:15:25,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0778ffa5d9f545d69d47c23a7e19575e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/0778ffa5d9f545d69d47c23a7e19575e 2024-11-10T18:15:25,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/71c30463a3164500ade8cde5b4a6b9e1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/71c30463a3164500ade8cde5b4a6b9e1 2024-11-10T18:15:25,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/57b002fdb3b8491a9f8cfe23e7d8fb63 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/57b002fdb3b8491a9f8cfe23e7d8fb63 2024-11-10T18:15:25,019 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01a71bcb5b7146f1bc66c7a2093d5273 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01a71bcb5b7146f1bc66c7a2093d5273 2024-11-10T18:15:25,020 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ca50f5f5915641869865e236df4a400a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/ca50f5f5915641869865e236df4a400a 2024-11-10T18:15:25,021 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/691ea56905d2:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01275a89f9d247748a72fb728da58da2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/01275a89f9d247748a72fb728da58da2 2024-11-10T18:15:25,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:25,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262585075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:25,086 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,087 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:25,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262585094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:25,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262585094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:25,195 INFO [Thread-1552 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1552. 2024-11-10T18:15:25,195 DEBUG [Thread-1552 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:25,195 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,195 DEBUG [Thread-1552 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,195 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:25,198 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-10T18:15:25,198 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:25,198 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,199 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,199 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:25,204 INFO [Thread-1550 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1550. 2024-11-10T18:15:25,204 DEBUG [Thread-1550 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:25,204 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,204 DEBUG [Thread-1550 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:25,207 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-10T18:15:25,207 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:25,207 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,207 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:25,209 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-10T18:15:25,209 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:25,209 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,209 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:25,209 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:25,227 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/8ebcbdb7aa584d50bfb9a3f89ed380ac 2024-11-10T18:15:25,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/3b590cff47914d4ba127148ebc053b43 is 50, key is test_row_0/B:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:25,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742234_1410 (size=12151) 2024-11-10T18:15:25,595 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39926 deadline: 1731262585595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,596 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:25,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:25,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39922 deadline: 1731262585615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39936 deadline: 1731262585615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:25,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:15:25,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/3b590cff47914d4ba127148ebc053b43 2024-11-10T18:15:25,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/9614f5b4c6bd4941aee0ef6606558067 is 50, key is test_row_0/C:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:25,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742235_1411 (size=12151) 2024-11-10T18:15:26,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/9614f5b4c6bd4941aee0ef6606558067 2024-11-10T18:15:26,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/8ebcbdb7aa584d50bfb9a3f89ed380ac as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac 2024-11-10T18:15:26,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac, entries=200, sequenceid=247, filesize=38.8 K 2024-11-10T18:15:26,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/3b590cff47914d4ba127148ebc053b43 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/3b590cff47914d4ba127148ebc053b43 2024-11-10T18:15:26,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/3b590cff47914d4ba127148ebc053b43, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T18:15:26,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/9614f5b4c6bd4941aee0ef6606558067 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9614f5b4c6bd4941aee0ef6606558067 2024-11-10T18:15:26,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9614f5b4c6bd4941aee0ef6606558067, entries=150, sequenceid=247, filesize=11.9 K 2024-11-10T18:15:26,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for c501ba03299c71686a077099ff2aacf7 in 1653ms, sequenceid=247, compaction requested=true 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:26,061 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c501ba03299c71686a077099ff2aacf7:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:26,061 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:26,061 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/A is initiating minor compaction (all files) 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/B is initiating minor compaction (all files) 2024-11-10T18:15:26,062 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/A in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:26,062 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/B in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:26,062 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/4a24aa525e9e4de5ba4160a6737ca34e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=100.1 K 2024-11-10T18:15:26,062 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/aac4967826234eaa87cbaf377b2592b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/1bcbe3df665d434d952ea86dc7ea4c7c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/3b590cff47914d4ba127148ebc053b43] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=36.1 K 2024-11-10T18:15:26,062 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/4a24aa525e9e4de5ba4160a6737ca34e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac] 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting aac4967826234eaa87cbaf377b2592b7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731262519772 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a24aa525e9e4de5ba4160a6737ca34e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731262519772 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bcbe3df665d434d952ea86dc7ea4c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731262520980 2024-11-10T18:15:26,062 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26e600eaabe949268c962853a5e6b5e0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731262520980 2024-11-10T18:15:26,063 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b590cff47914d4ba127148ebc053b43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731262523222 2024-11-10T18:15:26,063 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebcbdb7aa584d50bfb9a3f89ed380ac, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731262523219 2024-11-10T18:15:26,067 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:26,068 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#B#compaction#345 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:26,069 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/ce5ef6812c414d869469ddc68453248b is 50, key is test_row_0/B:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:26,069 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411102e7e4026cf2f470aa6e366069263887f_c501ba03299c71686a077099ff2aacf7 store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:26,072 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411102e7e4026cf2f470aa6e366069263887f_c501ba03299c71686a077099ff2aacf7, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:26,072 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e7e4026cf2f470aa6e366069263887f_c501ba03299c71686a077099ff2aacf7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:26,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742236_1412 (size=12254) 2024-11-10T18:15:26,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742237_1413 (size=4469) 2024-11-10T18:15:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-10T18:15:26,135 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:26,476 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#A#compaction#346 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:26,477 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/67afee72c8d9414ab02f049ede101984 is 175, key is test_row_0/A:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:26,479 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/ce5ef6812c414d869469ddc68453248b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/ce5ef6812c414d869469ddc68453248b 2024-11-10T18:15:26,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742238_1414 (size=31208) 2024-11-10T18:15:26,483 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/B of c501ba03299c71686a077099ff2aacf7 into ce5ef6812c414d869469ddc68453248b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:26,483 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:26,483 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/B, priority=13, startTime=1731262526061; duration=0sec 2024-11-10T18:15:26,483 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:26,483 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:B 2024-11-10T18:15:26,483 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:26,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:26,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): c501ba03299c71686a077099ff2aacf7/C is initiating minor compaction (all files) 2024-11-10T18:15:26,484 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c501ba03299c71686a077099ff2aacf7/C in TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:26,484 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/efadf3f2cec0447cb520d08af9149309, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9d70c41cbc854c57a1488db5ba6ad3c4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9614f5b4c6bd4941aee0ef6606558067] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp, totalSize=36.1 K 2024-11-10T18:15:26,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting efadf3f2cec0447cb520d08af9149309, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731262519772 2024-11-10T18:15:26,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d70c41cbc854c57a1488db5ba6ad3c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731262520980 2024-11-10T18:15:26,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 9614f5b4c6bd4941aee0ef6606558067, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731262523222 2024-11-10T18:15:26,490 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c501ba03299c71686a077099ff2aacf7#C#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:26,490 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/242bd10e458147639991b0c5c60e3f37 is 50, key is test_row_0/C:col10/1731262523233/Put/seqid=0 2024-11-10T18:15:26,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742239_1415 (size=12254) 2024-11-10T18:15:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:26,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T18:15:26,616 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-10T18:15:26,616 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:26,616 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:26,616 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:26,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:26,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:26,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a9527289d7384d91a87ab718778e58ac_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262524438/Put/seqid=0 2024-11-10T18:15:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742240_1416 (size=12454) 2024-11-10T18:15:26,626 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-10T18:15:26,626 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-11-10T18:15:26,626 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:26,626 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:26,626 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:26,626 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:26,626 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:26,626 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:26,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:26,626 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:26,884 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/67afee72c8d9414ab02f049ede101984 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/67afee72c8d9414ab02f049ede101984 2024-11-10T18:15:26,887 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/A of c501ba03299c71686a077099ff2aacf7 into 67afee72c8d9414ab02f049ede101984(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:26,887 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:26,887 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/A, priority=13, startTime=1731262526061; duration=0sec 2024-11-10T18:15:26,887 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:26,887 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:A 2024-11-10T18:15:26,896 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/242bd10e458147639991b0c5c60e3f37 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/242bd10e458147639991b0c5c60e3f37 2024-11-10T18:15:26,899 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c501ba03299c71686a077099ff2aacf7/C of c501ba03299c71686a077099ff2aacf7 into 242bd10e458147639991b0c5c60e3f37(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:26,900 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:26,900 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., storeName=c501ba03299c71686a077099ff2aacf7/C, priority=13, startTime=1731262526061; duration=0sec 2024-11-10T18:15:26,900 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:26,900 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c501ba03299c71686a077099ff2aacf7:C 2024-11-10T18:15:27,024 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:27,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a9527289d7384d91a87ab718778e58ac_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a9527289d7384d91a87ab718778e58ac_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:27,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/ecb5d1dc23dc412d9410bf41831eaa23, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:27,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/ecb5d1dc23dc412d9410bf41831eaa23 is 175, key is test_row_0/A:col10/1731262524438/Put/seqid=0 2024-11-10T18:15:27,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742241_1417 (size=31255) 2024-11-10T18:15:27,432 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/ecb5d1dc23dc412d9410bf41831eaa23 2024-11-10T18:15:27,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/2258917dd8ba47c3bf989d0ba72df40a is 50, key is test_row_0/B:col10/1731262524438/Put/seqid=0 2024-11-10T18:15:27,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742242_1418 (size=12301) 2024-11-10T18:15:27,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/2258917dd8ba47c3bf989d0ba72df40a 2024-11-10T18:15:27,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/f977653e4d4344c2a531f638ddcea699 is 50, key is test_row_0/C:col10/1731262524438/Put/seqid=0 2024-11-10T18:15:27,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742243_1419 (size=12301) 2024-11-10T18:15:28,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/f977653e4d4344c2a531f638ddcea699 2024-11-10T18:15:28,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/ecb5d1dc23dc412d9410bf41831eaa23 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/ecb5d1dc23dc412d9410bf41831eaa23 2024-11-10T18:15:28,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/ecb5d1dc23dc412d9410bf41831eaa23, entries=150, sequenceid=262, filesize=30.5 K 2024-11-10T18:15:28,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/2258917dd8ba47c3bf989d0ba72df40a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2258917dd8ba47c3bf989d0ba72df40a 2024-11-10T18:15:28,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2258917dd8ba47c3bf989d0ba72df40a, entries=150, sequenceid=262, filesize=12.0 K 2024-11-10T18:15:28,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/f977653e4d4344c2a531f638ddcea699 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f977653e4d4344c2a531f638ddcea699 2024-11-10T18:15:28,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f977653e4d4344c2a531f638ddcea699, entries=150, sequenceid=262, filesize=12.0 K 2024-11-10T18:15:28,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=13.42 KB/13740 for c501ba03299c71686a077099ff2aacf7 in 1649ms, sequenceid=262, compaction requested=false 2024-11-10T18:15:28,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c501ba03299c71686a077099ff2aacf7: 2024-11-10T18:15:31,786 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-11-10T18:15:31,786 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:31,786 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:31,786 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:31,786 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:34,217 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-10T18:15:34,217 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:34,217 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:34,217 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:34,217 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 8 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 31 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1043 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3126 rows 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1032 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3094 rows 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1043 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3125 rows 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1045 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3132 rows 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1041 2024-11-10T18:15:34,217 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3117 rows 2024-11-10T18:15:34,217 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:15:34,218 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:15:34,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:34,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:34,218 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:34,218 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T18:15:34,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T18:15:34,221 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262534221"}]},"ts":"1731262534221"} 2024-11-10T18:15:34,222 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T18:15:34,223 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T18:15:34,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:15:34,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, UNASSIGN}] 2024-11-10T18:15:34,225 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, UNASSIGN 2024-11-10T18:15:34,226 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:34,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=109, ppid=108, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, UNASSIGN because future has completed 2024-11-10T18:15:34,227 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:15:34,227 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:15:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T18:15:34,378 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(122): Close c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:34,378 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1722): Closing c501ba03299c71686a077099ff2aacf7, disabling compactions & flushes 2024-11-10T18:15:34,379 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. after waiting 0 ms 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:34,379 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(2902): Flushing c501ba03299c71686a077099ff2aacf7 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=A 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=B 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c501ba03299c71686a077099ff2aacf7, store=C 2024-11-10T18:15:34,379 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:34,384 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a571d8804d9a426f85ac5726027e8e04_c501ba03299c71686a077099ff2aacf7 is 50, key is test_row_0/A:col10/1731262534215/Put/seqid=0 2024-11-10T18:15:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742244_1420 (size=9914) 2024-11-10T18:15:34,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T18:15:34,788 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:34,791 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a571d8804d9a426f85ac5726027e8e04_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a571d8804d9a426f85ac5726027e8e04_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:34,792 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f3eb859fe99a4d1a848511ade2cdca03, store: [table=TestAcidGuarantees family=A region=c501ba03299c71686a077099ff2aacf7] 2024-11-10T18:15:34,792 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f3eb859fe99a4d1a848511ade2cdca03 is 175, key is test_row_0/A:col10/1731262534215/Put/seqid=0 2024-11-10T18:15:34,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742245_1421 (size=22561) 2024-11-10T18:15:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T18:15:35,196 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f3eb859fe99a4d1a848511ade2cdca03 2024-11-10T18:15:35,201 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/66188c9296f24ef0a95452b1d004998f is 50, key is test_row_0/B:col10/1731262534215/Put/seqid=0 2024-11-10T18:15:35,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742246_1422 (size=9857) 2024-11-10T18:15:35,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T18:15:35,605 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/66188c9296f24ef0a95452b1d004998f 2024-11-10T18:15:35,610 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/1a49151461c2413a829f0a35ba10e1a4 is 50, key is test_row_0/C:col10/1731262534215/Put/seqid=0 2024-11-10T18:15:35,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742247_1423 (size=9857) 2024-11-10T18:15:36,014 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/1a49151461c2413a829f0a35ba10e1a4 2024-11-10T18:15:36,017 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/A/f3eb859fe99a4d1a848511ade2cdca03 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f3eb859fe99a4d1a848511ade2cdca03 2024-11-10T18:15:36,020 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f3eb859fe99a4d1a848511ade2cdca03, entries=100, sequenceid=271, filesize=22.0 K 2024-11-10T18:15:36,021 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/B/66188c9296f24ef0a95452b1d004998f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/66188c9296f24ef0a95452b1d004998f 2024-11-10T18:15:36,024 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/66188c9296f24ef0a95452b1d004998f, entries=100, sequenceid=271, filesize=9.6 K 2024-11-10T18:15:36,024 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/.tmp/C/1a49151461c2413a829f0a35ba10e1a4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1a49151461c2413a829f0a35ba10e1a4 2024-11-10T18:15:36,027 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1a49151461c2413a829f0a35ba10e1a4, entries=100, sequenceid=271, filesize=9.6 K 2024-11-10T18:15:36,028 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c501ba03299c71686a077099ff2aacf7 in 1649ms, sequenceid=271, compaction requested=true 2024-11-10T18:15:36,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/4a24aa525e9e4de5ba4160a6737ca34e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac] to archive 2024-11-10T18:15:36,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:36,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/4a24aa525e9e4de5ba4160a6737ca34e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/4a24aa525e9e4de5ba4160a6737ca34e 2024-11-10T18:15:36,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/26e600eaabe949268c962853a5e6b5e0 2024-11-10T18:15:36,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/8ebcbdb7aa584d50bfb9a3f89ed380ac 2024-11-10T18:15:36,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/aac4967826234eaa87cbaf377b2592b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/1bcbe3df665d434d952ea86dc7ea4c7c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/3b590cff47914d4ba127148ebc053b43] to archive 2024-11-10T18:15:36,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:36,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/aac4967826234eaa87cbaf377b2592b7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/aac4967826234eaa87cbaf377b2592b7 2024-11-10T18:15:36,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/1bcbe3df665d434d952ea86dc7ea4c7c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/1bcbe3df665d434d952ea86dc7ea4c7c 2024-11-10T18:15:36,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/3b590cff47914d4ba127148ebc053b43 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/3b590cff47914d4ba127148ebc053b43 2024-11-10T18:15:36,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/efadf3f2cec0447cb520d08af9149309, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9d70c41cbc854c57a1488db5ba6ad3c4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9614f5b4c6bd4941aee0ef6606558067] to archive 2024-11-10T18:15:36,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:15:36,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/efadf3f2cec0447cb520d08af9149309 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/efadf3f2cec0447cb520d08af9149309 2024-11-10T18:15:36,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9d70c41cbc854c57a1488db5ba6ad3c4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9d70c41cbc854c57a1488db5ba6ad3c4 2024-11-10T18:15:36,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9614f5b4c6bd4941aee0ef6606558067 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/9614f5b4c6bd4941aee0ef6606558067 2024-11-10T18:15:36,043 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/recovered.edits/274.seqid, newMaxSeqId=274, maxSeqId=4 2024-11-10T18:15:36,044 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. 2024-11-10T18:15:36,044 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] regionserver.HRegion(1676): Region close journal for c501ba03299c71686a077099ff2aacf7: Waiting for close lock at 1731262534378Running coprocessor pre-close hooks at 1731262534378Disabling compacts and flushes for region at 1731262534378Disabling writes for close at 1731262534379 (+1 ms)Obtaining lock to block concurrent updates at 1731262534379Preparing flush snapshotting stores in c501ba03299c71686a077099ff2aacf7 at 1731262534379Finished memstore snapshotting TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731262534379Flushing stores of TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7. at 1731262534380 (+1 ms)Flushing c501ba03299c71686a077099ff2aacf7/A: creating writer at 1731262534380Flushing mob file c501ba03299c71686a077099ff2aacf7/A: appending metadata at 1731262534384 (+4 ms)Flushing mob file c501ba03299c71686a077099ff2aacf7/A: closing flushed file at 1731262534384Flushing c501ba03299c71686a077099ff2aacf7/A: appending metadata at 1731262534792 (+408 ms)Flushing c501ba03299c71686a077099ff2aacf7/A: closing flushed file at 1731262534792Flushing c501ba03299c71686a077099ff2aacf7/B: creating writer at 1731262535199 (+407 ms)Flushing c501ba03299c71686a077099ff2aacf7/B: appending metadata at 1731262535201 (+2 ms)Flushing c501ba03299c71686a077099ff2aacf7/B: closing flushed file at 1731262535201Flushing c501ba03299c71686a077099ff2aacf7/C: creating writer at 1731262535608 (+407 ms)Flushing c501ba03299c71686a077099ff2aacf7/C: appending metadata at 1731262535610 (+2 ms)Flushing c501ba03299c71686a077099ff2aacf7/C: closing flushed file at 1731262535610Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@357fbaa7: reopening flushed file at 1731262536017 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e979122: reopening flushed file at 1731262536020 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e141215: reopening flushed file at 1731262536024 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for c501ba03299c71686a077099ff2aacf7 in 1649ms, sequenceid=271, compaction requested=true at 1731262536028 (+4 ms)Writing region close event to WAL at 1731262536041 (+13 ms)Running coprocessor post-close hooks at 1731262536044 (+3 ms)Closed at 1731262536044 2024-11-10T18:15:36,046 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=110}] handler.UnassignRegionHandler(157): Closed c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,046 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=109 updating hbase:meta row=c501ba03299c71686a077099ff2aacf7, regionState=CLOSED 2024-11-10T18:15:36,047 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=110, ppid=109, state=RUNNABLE, hasLock=false; CloseRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:15:36,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-10T18:15:36,049 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseRegionProcedure c501ba03299c71686a077099ff2aacf7, server=691ea56905d2,32909,1731262402364 in 1.8210 sec 2024-11-10T18:15:36,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=109, resume processing ppid=108 2024-11-10T18:15:36,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, ppid=108, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c501ba03299c71686a077099ff2aacf7, UNASSIGN in 1.8250 sec 2024-11-10T18:15:36,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-10T18:15:36,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8280 sec 2024-11-10T18:15:36,054 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262536054"}]},"ts":"1731262536054"} 2024-11-10T18:15:36,055 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T18:15:36,055 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T18:15:36,057 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8370 sec 2024-11-10T18:15:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-10T18:15:36,365 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:36,366 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T18:15:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,367 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=111, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T18:15:36,368 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=111, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,370 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,372 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/recovered.edits] 2024-11-10T18:15:36,374 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/67afee72c8d9414ab02f049ede101984 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/67afee72c8d9414ab02f049ede101984 2024-11-10T18:15:36,375 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/ecb5d1dc23dc412d9410bf41831eaa23 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/ecb5d1dc23dc412d9410bf41831eaa23 2024-11-10T18:15:36,376 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f3eb859fe99a4d1a848511ade2cdca03 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/A/f3eb859fe99a4d1a848511ade2cdca03 2024-11-10T18:15:36,378 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2258917dd8ba47c3bf989d0ba72df40a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/2258917dd8ba47c3bf989d0ba72df40a 2024-11-10T18:15:36,379 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/66188c9296f24ef0a95452b1d004998f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/66188c9296f24ef0a95452b1d004998f 2024-11-10T18:15:36,379 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/ce5ef6812c414d869469ddc68453248b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/B/ce5ef6812c414d869469ddc68453248b 2024-11-10T18:15:36,381 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1a49151461c2413a829f0a35ba10e1a4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/1a49151461c2413a829f0a35ba10e1a4 2024-11-10T18:15:36,382 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/242bd10e458147639991b0c5c60e3f37 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/242bd10e458147639991b0c5c60e3f37 2024-11-10T18:15:36,383 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f977653e4d4344c2a531f638ddcea699 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/C/f977653e4d4344c2a531f638ddcea699 2024-11-10T18:15:36,385 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/recovered.edits/274.seqid to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7/recovered.edits/274.seqid 2024-11-10T18:15:36,385 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,385 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T18:15:36,386 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T18:15:36,386 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T18:15:36,389 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014abd008f49e4d3fb0ff8f65189f211e_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111014abd008f49e4d3fb0ff8f65189f211e_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,389 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111018ccc4174b104d26b8d1a3d11bccdd92_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111018ccc4174b104d26b8d1a3d11bccdd92_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,390 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111035c923b46da74ed3b69f091611a45504_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111035c923b46da74ed3b69f091611a45504_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,391 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105f414db5d1414d30b72c3f3148319be5_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411105f414db5d1414d30b72c3f3148319be5_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,392 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110633bf2afaf5e48e4935d3911b835e320_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110633bf2afaf5e48e4935d3911b835e320_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,393 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107034a4cbab7b43d694ec62497b1720cd_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411107034a4cbab7b43d694ec62497b1720cd_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,394 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a571d8804d9a426f85ac5726027e8e04_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a571d8804d9a426f85ac5726027e8e04_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,395 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a852bae48b3f4474afd9a260a7e7c979_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a852bae48b3f4474afd9a260a7e7c979_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,396 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a9527289d7384d91a87ab718778e58ac_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a9527289d7384d91a87ab718778e58ac_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,397 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ac05cf41fd284af79e9f383f7ad5bf4d_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ac05cf41fd284af79e9f383f7ad5bf4d_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,397 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2e8fbb304c64f478696e1a1b9e1d18a_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b2e8fbb304c64f478696e1a1b9e1d18a_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,398 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b68caaeba39e4f34bb2ae1669a036faa_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110b68caaeba39e4f34bb2ae1669a036faa_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,399 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2f10615544d4552a0e37a77ff5566b3_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2f10615544d4552a0e37a77ff5566b3_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,400 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d818b0aff13d491bab76d35c38fd6b34_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d818b0aff13d491bab76d35c38fd6b34_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,401 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dbda459cfca84eeaa5aa12eb2c283d3c_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dbda459cfca84eeaa5aa12eb2c283d3c_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e66901c6933349f3b3645e4b1a558eb2_c501ba03299c71686a077099ff2aacf7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110e66901c6933349f3b3645e4b1a558eb2_c501ba03299c71686a077099ff2aacf7 2024-11-10T18:15:36,402 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T18:15:36,404 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=111, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,405 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T18:15:36,407 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T18:15:36,408 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=111, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,408 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T18:15:36,408 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731262536408"}]},"ts":"9223372036854775807"} 2024-11-10T18:15:36,409 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T18:15:36,409 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c501ba03299c71686a077099ff2aacf7, NAME => 'TestAcidGuarantees,,1731262502095.c501ba03299c71686a077099ff2aacf7.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T18:15:36,409 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T18:15:36,410 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731262536410"}]},"ts":"9223372036854775807"} 2024-11-10T18:15:36,411 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T18:15:36,411 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=111, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 45 msec 2024-11-10T18:15:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-10T18:15:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T18:15:36,475 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:36,487 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 249), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 322), ProcessCount=11 (was 11), AvailableMemoryMB=8020 (was 8043) 2024-11-10T18:15:36,494 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=445, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=8019 2024-11-10T18:15:36,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:15:36,496 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:15:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T18:15:36,497 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T18:15:36,498 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:36,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 112 2024-11-10T18:15:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-10T18:15:36,498 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T18:15:36,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742248_1424 (size=1061) 2024-11-10T18:15:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-10T18:15:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-10T18:15:36,904 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:15:36,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742249_1425 (size=53) 2024-11-10T18:15:37,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-10T18:15:37,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:15:37,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6367e59850ed14d0f0996a4816a8c486, disabling compactions & flushes 2024-11-10T18:15:37,310 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. after waiting 0 ms 2024-11-10T18:15:37,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,310 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,310 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6367e59850ed14d0f0996a4816a8c486: Waiting for close lock at 1731262537309Disabling compacts and flushes for region at 1731262537309Disabling writes for close at 1731262537310 (+1 ms)Writing region close event to WAL at 1731262537310Closed at 1731262537310 2024-11-10T18:15:37,311 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T18:15:37,311 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731262537311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731262537311"}]},"ts":"1731262537311"} 2024-11-10T18:15:37,312 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T18:15:37,312 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T18:15:37,313 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262537313"}]},"ts":"1731262537313"} 2024-11-10T18:15:37,314 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T18:15:37,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, ASSIGN}] 2024-11-10T18:15:37,315 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, ASSIGN 2024-11-10T18:15:37,315 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, ASSIGN; state=OFFLINE, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=false 2024-11-10T18:15:37,466 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6367e59850ed14d0f0996a4816a8c486, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:37,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, ASSIGN because future has completed 2024-11-10T18:15:37,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:15:37,622 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,622 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7752): Opening region: {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:15:37,623 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,623 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:15:37,623 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7794): checking encryption for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,623 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(7797): checking classloading for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,624 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,625 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:37,625 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6367e59850ed14d0f0996a4816a8c486 columnFamilyName A 2024-11-10T18:15:37,625 DEBUG [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:37,626 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.HStore(327): Store=6367e59850ed14d0f0996a4816a8c486/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:37,626 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,627 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:37,627 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6367e59850ed14d0f0996a4816a8c486 columnFamilyName B 2024-11-10T18:15:37,627 DEBUG [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:37,627 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.HStore(327): Store=6367e59850ed14d0f0996a4816a8c486/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:37,627 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,628 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:15:37,628 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6367e59850ed14d0f0996a4816a8c486 columnFamilyName C 2024-11-10T18:15:37,628 DEBUG [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:15:37,628 INFO [StoreOpener-6367e59850ed14d0f0996a4816a8c486-1 {}] regionserver.HStore(327): Store=6367e59850ed14d0f0996a4816a8c486/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:15:37,629 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,629 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1038): replaying wal for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,629 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,629 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,630 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1048): stopping wal replay for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,630 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1060): Cleaning up temporary data for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,630 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:15:37,631 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1093): writing seq id for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,633 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:15:37,633 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1114): Opened 6367e59850ed14d0f0996a4816a8c486; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72688854, jitterRate=0.08314833045005798}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:15:37,633 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:37,634 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegion(1006): Region open journal for 6367e59850ed14d0f0996a4816a8c486: Running coprocessor pre-open hook at 1731262537623Writing region info on filesystem at 1731262537623Initializing all the Stores at 1731262537624 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262537624Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262537624Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262537624Cleaning up temporary data from old regions at 1731262537630 (+6 ms)Running coprocessor post-open hooks at 1731262537633 (+3 ms)Region opened successfully at 1731262537634 (+1 ms) 2024-11-10T18:15:37,634 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., pid=114, masterSystemTime=1731262537620 2024-11-10T18:15:37,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-10T18:15:37,636 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,636 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=114}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:37,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=6367e59850ed14d0f0996a4816a8c486, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:15:37,637 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:15:37,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-10T18:15:37,639 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; OpenRegionProcedure 6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 in 170 msec 2024-11-10T18:15:37,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-10T18:15:37,640 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, ASSIGN in 325 msec 2024-11-10T18:15:37,641 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T18:15:37,641 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262537641"}]},"ts":"1731262537641"} 2024-11-10T18:15:37,642 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T18:15:37,643 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=112, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T18:15:37,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1470 sec 2024-11-10T18:15:38,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-11-10T18:15:38,645 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:38,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d1f058, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e315cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,647 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,648 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e42cbca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,649 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,650 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,651 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@42b36682 2024-11-10T18:15:38,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:15:38,653 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:15:38,654 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,654 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,654 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d08ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,656 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b413993, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,656 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,656 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,657 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,657 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21e911c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,658 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c4db0dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,660 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@538346ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,661 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215851db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,662 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,662 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,662 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,662 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463b53eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,663 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59092eee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fc361f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,665 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d8a369f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,666 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@656b2291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,667 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,667 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,667 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,667 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35fa4ed0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,667 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,668 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49082, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ffdbe6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,670 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,671 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d76135, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,671 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,672 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,672 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753a5628, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,673 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226b5665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,674 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,674 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49130, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,675 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0833e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,676 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b782266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,677 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49146, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,677 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43df4d67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ca199c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:15:38,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:15:38,678 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:15:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:15:38,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b839be9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:15:38,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:38,679 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:15:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60206d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:15:38,680 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,680 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,680 DEBUG [Thread-1938 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,680 DEBUG [Thread-1923 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,681 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,682 DEBUG [Thread-1942 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,682 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-10T18:15:38,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35152, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,682 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,682 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,682 DEBUG [Thread-1940 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T18:15:38,683 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35138, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,684 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,685 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:15:38,686 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35236, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:15:38,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,688 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,689 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:38,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2] 2024-11-10T18:15:38,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:38,694 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:38,694 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:38,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/7e76feae7e2c4b96be7204ecdf47f673 is 50, key is test_row_1/A:col10/1731262538691/Put/seqid=0 2024-11-10T18:15:38,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742250_1426 (size=11997) 2024-11-10T18:15:38,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262598722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262598722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262598730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262538718, max timestamp=1731262538729Num uniques 859; 2024-11-10T18:15:38,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:38,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262538718, max timestamp=1731262538729Num uniques 859; 2024-11-10T18:15:38,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262538718, max timestamp=1731262538729Num uniques 859; 2024-11-10T18:15:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,734 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:38,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T18:15:38,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262598790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262598795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262598834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T18:15:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:38,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:38,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:38,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262598844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262598845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,857 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262598904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,912 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:38,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262598914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:38,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T18:15:38,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T18:15:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:38,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:38,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:39,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262599054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262599055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262599065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/7e76feae7e2c4b96be7204ecdf47f673 2024-11-10T18:15:39,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262599124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262599135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,145 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T18:15:39,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:39,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:39,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:39,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:39,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:39,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cccc080998bb4cdbaaeca0d38063afe1 is 50, key is test_row_1/B:col10/1731262538691/Put/seqid=0 2024-11-10T18:15:39,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742251_1427 (size=9657) 2024-11-10T18:15:39,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cccc080998bb4cdbaaeca0d38063afe1 2024-11-10T18:15:39,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/5aad43413be44db69ac1e65a3dac0b35 is 50, key is test_row_1/C:col10/1731262538691/Put/seqid=0 2024-11-10T18:15:39,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742252_1428 (size=9657) 2024-11-10T18:15:39,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/5aad43413be44db69ac1e65a3dac0b35 2024-11-10T18:15:39,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/7e76feae7e2c4b96be7204ecdf47f673 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/7e76feae7e2c4b96be7204ecdf47f673 2024-11-10T18:15:39,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/7e76feae7e2c4b96be7204ecdf47f673, entries=150, sequenceid=11, filesize=11.7 K 2024-11-10T18:15:39,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cccc080998bb4cdbaaeca0d38063afe1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cccc080998bb4cdbaaeca0d38063afe1 2024-11-10T18:15:39,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cccc080998bb4cdbaaeca0d38063afe1, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T18:15:39,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/5aad43413be44db69ac1e65a3dac0b35 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/5aad43413be44db69ac1e65a3dac0b35 2024-11-10T18:15:39,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/5aad43413be44db69ac1e65a3dac0b35, entries=100, sequenceid=11, filesize=9.4 K 2024-11-10T18:15:39,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6367e59850ed14d0f0996a4816a8c486 in 514ms, sequenceid=11, compaction requested=false 2024-11-10T18:15:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T18:15:39,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:39,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-10T18:15:39,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:39,302 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T18:15:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:39,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T18:15:39,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/def3c05868954003bb7f413908f020c2 is 50, key is test_row_0/A:col10/1731262538794/Put/seqid=0 2024-11-10T18:15:39,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742253_1429 (size=12001) 2024-11-10T18:15:39,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:39,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:39,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262599397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262599398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262599399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262599435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262599455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262599504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262599515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262599515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,712 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/def3c05868954003bb7f413908f020c2 2024-11-10T18:15:39,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/f32f1ce94e7f4b81879ac9e55ca17682 is 50, key is test_row_0/B:col10/1731262538794/Put/seqid=0 2024-11-10T18:15:39,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742254_1430 (size=12001) 2024-11-10T18:15:39,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262599714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,726 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262599735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262599735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T18:15:39,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262599955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:39,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262599975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262600034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,040 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262600055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262600055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,124 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/f32f1ce94e7f4b81879ac9e55ca17682 2024-11-10T18:15:40,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/a57f7a0f97974f509cf31db4c1ef0b5d is 50, key is test_row_0/C:col10/1731262538794/Put/seqid=0 2024-11-10T18:15:40,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742255_1431 (size=12001) 2024-11-10T18:15:40,135 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/a57f7a0f97974f509cf31db4c1ef0b5d 2024-11-10T18:15:40,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/def3c05868954003bb7f413908f020c2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/def3c05868954003bb7f413908f020c2 2024-11-10T18:15:40,143 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/def3c05868954003bb7f413908f020c2, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T18:15:40,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/f32f1ce94e7f4b81879ac9e55ca17682 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/f32f1ce94e7f4b81879ac9e55ca17682 2024-11-10T18:15:40,147 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/f32f1ce94e7f4b81879ac9e55ca17682, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T18:15:40,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/a57f7a0f97974f509cf31db4c1ef0b5d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a57f7a0f97974f509cf31db4c1ef0b5d 2024-11-10T18:15:40,151 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a57f7a0f97974f509cf31db4c1ef0b5d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-10T18:15:40,152 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6367e59850ed14d0f0996a4816a8c486 in 850ms, sequenceid=48, compaction requested=false 2024-11-10T18:15:40,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:40,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:40,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-10T18:15:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-10T18:15:40,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-10T18:15:40,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4640 sec 2024-11-10T18:15:40,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4750 sec 2024-11-10T18:15:40,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:40,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:15:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:40,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:40,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e709000c94ac47bba8849f74ea9c615c is 50, key is test_row_0/A:col10/1731262539383/Put/seqid=0 2024-11-10T18:15:40,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742256_1432 (size=14341) 2024-11-10T18:15:40,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e709000c94ac47bba8849f74ea9c615c 2024-11-10T18:15:40,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/389e48079cb344b5bada874a17e898b1 is 50, key is test_row_0/B:col10/1731262539383/Put/seqid=0 2024-11-10T18:15:40,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742257_1433 (size=12001) 2024-11-10T18:15:40,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262600656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262600656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731262540638, max timestamp=1731262540656Num uniques 859; 2024-11-10T18:15:40,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731262540638, max timestamp=1731262540656Num uniques 859; 2024-11-10T18:15:40,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731262540638, max timestamp=1731262540656Num uniques 859; 2024-11-10T18:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T18:15:40,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T18:15:40,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-10T18:15:40,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262600765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262600766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262600777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-10T18:15:40,825 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:40,825 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-10T18:15:40,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:40,827 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:40,828 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:40,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262600885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262600885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,892 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:40,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:40,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:40,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:40,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262600974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,986 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/389e48079cb344b5bada874a17e898b1 2024-11-10T18:15:40,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262600984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:40,996 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:40,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/dfff82fc47c24d418952a15387866b27 is 50, key is test_row_0/C:col10/1731262539383/Put/seqid=0 2024-11-10T18:15:41,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742258_1434 (size=12001) 2024-11-10T18:15:41,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262601004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,012 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,013 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262601094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262601095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:41,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:41,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:41,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262601304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/dfff82fc47c24d418952a15387866b27 2024-11-10T18:15:41,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e709000c94ac47bba8849f74ea9c615c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e709000c94ac47bba8849f74ea9c615c 2024-11-10T18:15:41,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e709000c94ac47bba8849f74ea9c615c, entries=200, sequenceid=59, filesize=14.0 K 2024-11-10T18:15:41,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/389e48079cb344b5bada874a17e898b1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/389e48079cb344b5bada874a17e898b1 2024-11-10T18:15:41,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/389e48079cb344b5bada874a17e898b1, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T18:15:41,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/dfff82fc47c24d418952a15387866b27 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/dfff82fc47c24d418952a15387866b27 2024-11-10T18:15:41,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/dfff82fc47c24d418952a15387866b27, entries=150, sequenceid=59, filesize=11.7 K 2024-11-10T18:15:41,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6367e59850ed14d0f0996a4816a8c486 in 858ms, sequenceid=59, compaction requested=true 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:41,417 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:41,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:41,417 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:41,418 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:41,418 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:41,418 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:41,418 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:41,418 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,418 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,418 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/7e76feae7e2c4b96be7204ecdf47f673, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/def3c05868954003bb7f413908f020c2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e709000c94ac47bba8849f74ea9c615c] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=37.4 K 2024-11-10T18:15:41,418 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cccc080998bb4cdbaaeca0d38063afe1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/f32f1ce94e7f4b81879ac9e55ca17682, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/389e48079cb344b5bada874a17e898b1] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=32.9 K 2024-11-10T18:15:41,419 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cccc080998bb4cdbaaeca0d38063afe1, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731262538691 2024-11-10T18:15:41,419 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e76feae7e2c4b96be7204ecdf47f673, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731262538691 2024-11-10T18:15:41,419 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f32f1ce94e7f4b81879ac9e55ca17682, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731262538776 2024-11-10T18:15:41,419 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting def3c05868954003bb7f413908f020c2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731262538776 2024-11-10T18:15:41,419 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 389e48079cb344b5bada874a17e898b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731262539383 2024-11-10T18:15:41,419 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e709000c94ac47bba8849f74ea9c615c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731262539383 2024-11-10T18:15:41,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:41,426 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-10T18:15:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:41,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:41,435 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:41,435 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:41,435 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e0d9daee4965492a88737a1085fbc13f is 50, key is test_row_0/A:col10/1731262539383/Put/seqid=0 2024-11-10T18:15:41,435 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/520cdd3308164f73a88e247ced7b5103 is 50, key is test_row_0/B:col10/1731262539383/Put/seqid=0 2024-11-10T18:15:41,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/cdf73db9b67b45fc91f020ffa755ff2b is 50, key is test_row_0/A:col10/1731262540758/Put/seqid=0 2024-11-10T18:15:41,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742259_1435 (size=12104) 2024-11-10T18:15:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:41,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742260_1436 (size=12104) 2024-11-10T18:15:41,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742261_1437 (size=14341) 2024-11-10T18:15:41,449 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/520cdd3308164f73a88e247ced7b5103 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/520cdd3308164f73a88e247ced7b5103 2024-11-10T18:15:41,453 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into 520cdd3308164f73a88e247ced7b5103(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:41,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:41,453 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=13, startTime=1731262541417; duration=0sec 2024-11-10T18:15:41,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:41,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:41,453 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:41,454 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:41,454 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:41,454 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,454 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/5aad43413be44db69ac1e65a3dac0b35, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a57f7a0f97974f509cf31db4c1ef0b5d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/dfff82fc47c24d418952a15387866b27] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=32.9 K 2024-11-10T18:15:41,455 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aad43413be44db69ac1e65a3dac0b35, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731262538691 2024-11-10T18:15:41,455 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a57f7a0f97974f509cf31db4c1ef0b5d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731262538776 2024-11-10T18:15:41,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:41,455 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting dfff82fc47c24d418952a15387866b27, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731262539383 2024-11-10T18:15:41,463 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:41,463 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/c7dab47031a846c4ae263330215c643f is 50, key is test_row_0/C:col10/1731262539383/Put/seqid=0 2024-11-10T18:15:41,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742262_1438 (size=12104) 2024-11-10T18:15:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262601458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262601464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,558 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:15:41,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:41,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262601585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262601585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:41,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262601805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262601805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262601824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:41,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:41,844 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e0d9daee4965492a88737a1085fbc13f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e0d9daee4965492a88737a1085fbc13f 2024-11-10T18:15:41,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/cdf73db9b67b45fc91f020ffa755ff2b 2024-11-10T18:15:41,848 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into e0d9daee4965492a88737a1085fbc13f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:41,848 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:41,848 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=13, startTime=1731262541417; duration=0sec 2024-11-10T18:15:41,848 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:41,848 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:41,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6307276970224e6e9a681db767bfd6ba is 50, key is test_row_0/B:col10/1731262540758/Put/seqid=0 2024-11-10T18:15:41,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742263_1439 (size=12001) 2024-11-10T18:15:41,873 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/c7dab47031a846c4ae263330215c643f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c7dab47031a846c4ae263330215c643f 2024-11-10T18:15:41,876 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into c7dab47031a846c4ae263330215c643f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:41,876 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:41,877 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=13, startTime=1731262541417; duration=0sec 2024-11-10T18:15:41,877 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:41,877 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:41,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:41,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:41,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:41,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:41,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:42,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:42,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:42,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:42,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:42,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:42,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262602125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262602125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:42,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:42,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:42,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:42,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6307276970224e6e9a681db767bfd6ba 2024-11-10T18:15:42,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/f469f8526bcd49f9819f5d7016a5372b is 50, key is test_row_0/C:col10/1731262540758/Put/seqid=0 2024-11-10T18:15:42,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742264_1440 (size=12001) 2024-11-10T18:15:42,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/f469f8526bcd49f9819f5d7016a5372b 2024-11-10T18:15:42,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/cdf73db9b67b45fc91f020ffa755ff2b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/cdf73db9b67b45fc91f020ffa755ff2b 2024-11-10T18:15:42,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/cdf73db9b67b45fc91f020ffa755ff2b, entries=200, sequenceid=94, filesize=14.0 K 2024-11-10T18:15:42,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6307276970224e6e9a681db767bfd6ba as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6307276970224e6e9a681db767bfd6ba 2024-11-10T18:15:42,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6307276970224e6e9a681db767bfd6ba, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T18:15:42,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/f469f8526bcd49f9819f5d7016a5372b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f469f8526bcd49f9819f5d7016a5372b 2024-11-10T18:15:42,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f469f8526bcd49f9819f5d7016a5372b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-10T18:15:42,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 6367e59850ed14d0f0996a4816a8c486 in 876ms, sequenceid=94, compaction requested=false 2024-11-10T18:15:42,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:42,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-10T18:15:42,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:42,357 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:42,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:42,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/8bcea8cb6d7d4734aff835f32bccc066 is 50, key is test_row_0/A:col10/1731262541463/Put/seqid=0 2024-11-10T18:15:42,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742265_1441 (size=12001) 2024-11-10T18:15:42,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:42,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:42,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262542693, max timestamp=1731262542711Num uniques 859; 2024-11-10T18:15:42,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262542693, max timestamp=1731262542711Num uniques 859; 2024-11-10T18:15:42,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:42,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:42,715 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262542693, max timestamp=1731262542711Num uniques 859; 2024-11-10T18:15:42,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:42,766 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/8bcea8cb6d7d4734aff835f32bccc066 2024-11-10T18:15:42,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cbbab48f484a4caeb2066b9ca1c34eb4 is 50, key is test_row_0/B:col10/1731262541463/Put/seqid=0 2024-11-10T18:15:42,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742266_1442 (size=12001) 2024-11-10T18:15:42,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262602790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262602795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262602854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,865 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262602905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262602914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:42,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:42,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:43,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:43,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262602995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:43,002 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,002 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:43,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:43,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262603014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:43,025 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,026 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:43,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:43,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262603124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:43,143 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:43,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262603134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:43,143 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:43,176 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cbbab48f484a4caeb2066b9ca1c34eb4 2024-11-10T18:15:43,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/2507dda96cf443f1b1f89ff6dfddf01d is 50, key is test_row_0/C:col10/1731262541463/Put/seqid=0 2024-11-10T18:15:43,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742267_1443 (size=12001) 2024-11-10T18:15:43,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:43,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262603445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:43,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262603455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:43,587 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/2507dda96cf443f1b1f89ff6dfddf01d 2024-11-10T18:15:43,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/8bcea8cb6d7d4734aff835f32bccc066 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8bcea8cb6d7d4734aff835f32bccc066 2024-11-10T18:15:43,595 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8bcea8cb6d7d4734aff835f32bccc066, entries=150, sequenceid=107, filesize=11.7 K 2024-11-10T18:15:43,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cbbab48f484a4caeb2066b9ca1c34eb4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cbbab48f484a4caeb2066b9ca1c34eb4 2024-11-10T18:15:43,599 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cbbab48f484a4caeb2066b9ca1c34eb4, entries=150, sequenceid=107, filesize=11.7 K 2024-11-10T18:15:43,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/2507dda96cf443f1b1f89ff6dfddf01d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2507dda96cf443f1b1f89ff6dfddf01d 2024-11-10T18:15:43,603 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2507dda96cf443f1b1f89ff6dfddf01d, entries=150, sequenceid=107, filesize=11.7 K 2024-11-10T18:15:43,603 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6367e59850ed14d0f0996a4816a8c486 in 1246ms, sequenceid=107, compaction requested=true 2024-11-10T18:15:43,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:43,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:43,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-10T18:15:43,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-10T18:15:43,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-10T18:15:43,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7770 sec 2024-11-10T18:15:43,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.7820 sec 2024-11-10T18:15:43,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:43,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T18:15:43,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:43,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:43,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:43,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:43,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:43,971 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:43,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/6e0718ee4bad4d3789c4d0b87e0ff4bf is 50, key is test_row_0/A:col10/1731262542794/Put/seqid=0 2024-11-10T18:15:43,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742268_1444 (size=14541) 2024-11-10T18:15:44,007 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262603994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262603998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262604115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262604115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262604325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262604335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/6e0718ee4bad4d3789c4d0b87e0ff4bf 2024-11-10T18:15:44,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cb73cb4f6b864884aecf0e6b4106752f is 50, key is test_row_0/B:col10/1731262542794/Put/seqid=0 2024-11-10T18:15:44,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742269_1445 (size=12151) 2024-11-10T18:15:44,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262604645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262604645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cb73cb4f6b864884aecf0e6b4106752f 2024-11-10T18:15:44,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/83d1ddaa9a9f4868a37ccb3423ee23dc is 50, key is test_row_0/C:col10/1731262542794/Put/seqid=0 2024-11-10T18:15:44,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742270_1446 (size=12151) 2024-11-10T18:15:44,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:44,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262604894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:44,907 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:44,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:44,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-10T18:15:44,985 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:44,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:44,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-10T18:15:44,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T18:15:44,987 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:44,988 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:44,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T18:15:45,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T18:15:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:45,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:45,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:45,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:45,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:45,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262605164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262605165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:45,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/83d1ddaa9a9f4868a37ccb3423ee23dc 2024-11-10T18:15:45,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/6e0718ee4bad4d3789c4d0b87e0ff4bf as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/6e0718ee4bad4d3789c4d0b87e0ff4bf 2024-11-10T18:15:45,208 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/6e0718ee4bad4d3789c4d0b87e0ff4bf, entries=200, sequenceid=144, filesize=14.2 K 2024-11-10T18:15:45,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/cb73cb4f6b864884aecf0e6b4106752f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cb73cb4f6b864884aecf0e6b4106752f 2024-11-10T18:15:45,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cb73cb4f6b864884aecf0e6b4106752f, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T18:15:45,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/83d1ddaa9a9f4868a37ccb3423ee23dc as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/83d1ddaa9a9f4868a37ccb3423ee23dc 2024-11-10T18:15:45,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/83d1ddaa9a9f4868a37ccb3423ee23dc, entries=150, sequenceid=144, filesize=11.9 K 2024-11-10T18:15:45,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 6367e59850ed14d0f0996a4816a8c486 in 1246ms, sequenceid=144, compaction requested=true 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:45,216 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:45,216 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:45,216 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:45,219 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:45,219 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:45,219 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/520cdd3308164f73a88e247ced7b5103, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6307276970224e6e9a681db767bfd6ba, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cbbab48f484a4caeb2066b9ca1c34eb4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cb73cb4f6b864884aecf0e6b4106752f] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=47.1 K 2024-11-10T18:15:45,219 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e0d9daee4965492a88737a1085fbc13f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/cdf73db9b67b45fc91f020ffa755ff2b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8bcea8cb6d7d4734aff835f32bccc066, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/6e0718ee4bad4d3789c4d0b87e0ff4bf] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=51.7 K 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 520cdd3308164f73a88e247ced7b5103, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731262539383 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0d9daee4965492a88737a1085fbc13f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731262539383 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6307276970224e6e9a681db767bfd6ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731262540758 2024-11-10T18:15:45,219 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdf73db9b67b45fc91f020ffa755ff2b, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731262540758 2024-11-10T18:15:45,220 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cbbab48f484a4caeb2066b9ca1c34eb4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731262541438 2024-11-10T18:15:45,220 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcea8cb6d7d4734aff835f32bccc066, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731262541438 2024-11-10T18:15:45,220 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cb73cb4f6b864884aecf0e6b4106752f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731262542777 2024-11-10T18:15:45,220 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e0718ee4bad4d3789c4d0b87e0ff4bf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731262542777 2024-11-10T18:15:45,228 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#375 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:45,228 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/9e25497d425e451f97bc00433c65f5e7 is 50, key is test_row_0/A:col10/1731262542794/Put/seqid=0 2024-11-10T18:15:45,231 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:45,232 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/d1249eecd80544abaa664ff7d2d51f14 is 50, key is test_row_0/B:col10/1731262542794/Put/seqid=0 2024-11-10T18:15:45,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742272_1448 (size=12391) 2024-11-10T18:15:45,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742271_1447 (size=12391) 2024-11-10T18:15:45,251 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/9e25497d425e451f97bc00433c65f5e7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9e25497d425e451f97bc00433c65f5e7 2024-11-10T18:15:45,251 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/d1249eecd80544abaa664ff7d2d51f14 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/d1249eecd80544abaa664ff7d2d51f14 2024-11-10T18:15:45,256 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into d1249eecd80544abaa664ff7d2d51f14(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:45,256 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=12, startTime=1731262545216; duration=0sec 2024-11-10T18:15:45,256 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into 9e25497d425e451f97bc00433c65f5e7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:45,256 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=12, startTime=1731262545216; duration=0sec 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:45,256 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:45,258 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:45,258 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:45,258 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:45,258 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c7dab47031a846c4ae263330215c643f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f469f8526bcd49f9819f5d7016a5372b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2507dda96cf443f1b1f89ff6dfddf01d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/83d1ddaa9a9f4868a37ccb3423ee23dc] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=47.1 K 2024-11-10T18:15:45,258 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c7dab47031a846c4ae263330215c643f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731262539383 2024-11-10T18:15:45,259 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f469f8526bcd49f9819f5d7016a5372b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731262540758 2024-11-10T18:15:45,259 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 2507dda96cf443f1b1f89ff6dfddf01d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731262541438 2024-11-10T18:15:45,259 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 83d1ddaa9a9f4868a37ccb3423ee23dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731262542777 2024-11-10T18:15:45,268 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:45,268 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/0fc0656368174ab5aa4449a1d694f17e is 50, key is test_row_0/C:col10/1731262542794/Put/seqid=0 2024-11-10T18:15:45,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742273_1449 (size=12391) 2024-11-10T18:15:45,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-10T18:15:45,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:45,294 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-10T18:15:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:45,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:45,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e4c775a661ec4426a79cf1eee3c7f06c is 50, key is test_row_0/A:col10/1731262543993/Put/seqid=0 2024-11-10T18:15:45,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742274_1450 (size=12151) 2024-11-10T18:15:45,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T18:15:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T18:15:45,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/0fc0656368174ab5aa4449a1d694f17e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0fc0656368174ab5aa4449a1d694f17e 2024-11-10T18:15:45,685 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into 0fc0656368174ab5aa4449a1d694f17e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:45,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:45,685 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=12, startTime=1731262545216; duration=0sec 2024-11-10T18:15:45,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:45,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:45,703 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e4c775a661ec4426a79cf1eee3c7f06c 2024-11-10T18:15:45,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/7e4687c7c5ba424c95d1f904768f8db0 is 50, key is test_row_0/B:col10/1731262543993/Put/seqid=0 2024-11-10T18:15:45,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742275_1451 (size=12151) 2024-11-10T18:15:45,716 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/7e4687c7c5ba424c95d1f904768f8db0 2024-11-10T18:15:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/954cb0ad30ab4ac68e1ea68f88ee3b42 is 50, key is test_row_0/C:col10/1731262543993/Put/seqid=0 2024-11-10T18:15:45,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742276_1452 (size=12151) 2024-11-10T18:15:46,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T18:15:46,135 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/954cb0ad30ab4ac68e1ea68f88ee3b42 2024-11-10T18:15:46,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/e4c775a661ec4426a79cf1eee3c7f06c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e4c775a661ec4426a79cf1eee3c7f06c 2024-11-10T18:15:46,143 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e4c775a661ec4426a79cf1eee3c7f06c, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T18:15:46,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/7e4687c7c5ba424c95d1f904768f8db0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/7e4687c7c5ba424c95d1f904768f8db0 2024-11-10T18:15:46,155 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/7e4687c7c5ba424c95d1f904768f8db0, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T18:15:46,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/954cb0ad30ab4ac68e1ea68f88ee3b42 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/954cb0ad30ab4ac68e1ea68f88ee3b42 2024-11-10T18:15:46,159 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/954cb0ad30ab4ac68e1ea68f88ee3b42, entries=150, sequenceid=155, filesize=11.9 K 2024-11-10T18:15:46,160 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 6367e59850ed14d0f0996a4816a8c486 in 867ms, sequenceid=155, compaction requested=false 2024-11-10T18:15:46,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:46,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:46,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-10T18:15:46,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-10T18:15:46,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-10T18:15:46,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1740 sec 2024-11-10T18:15:46,166 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.1790 sec 2024-11-10T18:15:46,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:46,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:46,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:46,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/8c51115b8508444cbb010d10608ee237 is 50, key is test_row_0/A:col10/1731262546188/Put/seqid=0 2024-11-10T18:15:46,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742277_1453 (size=16931) 2024-11-10T18:15:46,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262546223, max timestamp=1731262546265Num uniques 859; 2024-11-10T18:15:46,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262546223, max timestamp=1731262546265Num uniques 859; 2024-11-10T18:15:46,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:46,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:46,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262546223, max timestamp=1731262546265Num uniques 859; 2024-11-10T18:15:46,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:46,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262606348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:46,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262606349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:46,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262606464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:46,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262606474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:46,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/8c51115b8508444cbb010d10608ee237 2024-11-10T18:15:46,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/4aabfe18b0f34eb392a53436e0381831 is 50, key is test_row_0/B:col10/1731262546188/Put/seqid=0 2024-11-10T18:15:46,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742278_1454 (size=12151) 2024-11-10T18:15:46,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:46,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262606675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:46,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:46,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262606694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:46,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262606995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/4aabfe18b0f34eb392a53436e0381831 2024-11-10T18:15:47,013 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d5952e038f68445f90470bb0f835cfbe is 50, key is test_row_0/C:col10/1731262546188/Put/seqid=0 2024-11-10T18:15:47,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742279_1455 (size=12151) 2024-11-10T18:15:47,020 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d5952e038f68445f90470bb0f835cfbe 2024-11-10T18:15:47,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/8c51115b8508444cbb010d10608ee237 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8c51115b8508444cbb010d10608ee237 2024-11-10T18:15:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262607015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8c51115b8508444cbb010d10608ee237, entries=250, sequenceid=166, filesize=16.5 K 2024-11-10T18:15:47,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262607015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,027 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/4aabfe18b0f34eb392a53436e0381831 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/4aabfe18b0f34eb392a53436e0381831 2024-11-10T18:15:47,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/4aabfe18b0f34eb392a53436e0381831, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T18:15:47,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d5952e038f68445f90470bb0f835cfbe as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d5952e038f68445f90470bb0f835cfbe 2024-11-10T18:15:47,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d5952e038f68445f90470bb0f835cfbe, entries=150, sequenceid=166, filesize=11.9 K 2024-11-10T18:15:47,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6367e59850ed14d0f0996a4816a8c486 in 853ms, sequenceid=166, compaction requested=true 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:47,042 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:47,042 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:47,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:47,043 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:47,043 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:47,043 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:47,043 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,043 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:47,043 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,043 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/d1249eecd80544abaa664ff7d2d51f14, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/7e4687c7c5ba424c95d1f904768f8db0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/4aabfe18b0f34eb392a53436e0381831] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=35.8 K 2024-11-10T18:15:47,043 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9e25497d425e451f97bc00433c65f5e7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e4c775a661ec4426a79cf1eee3c7f06c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8c51115b8508444cbb010d10608ee237] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=40.5 K 2024-11-10T18:15:47,044 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d1249eecd80544abaa664ff7d2d51f14, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731262542777 2024-11-10T18:15:47,044 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e25497d425e451f97bc00433c65f5e7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731262542777 2024-11-10T18:15:47,045 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e4687c7c5ba424c95d1f904768f8db0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731262543983 2024-11-10T18:15:47,045 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4c775a661ec4426a79cf1eee3c7f06c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731262543983 2024-11-10T18:15:47,045 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 4aabfe18b0f34eb392a53436e0381831, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731262546182 2024-11-10T18:15:47,045 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c51115b8508444cbb010d10608ee237, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731262546177 2024-11-10T18:15:47,055 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#384 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:47,056 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/ee69d028962646d381c6a48a0402d1fb is 50, key is test_row_0/B:col10/1731262546188/Put/seqid=0 2024-11-10T18:15:47,059 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#385 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:47,059 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/84fab6e45c95414ba9fdcf1c9e2b7568 is 50, key is test_row_0/A:col10/1731262546188/Put/seqid=0 2024-11-10T18:15:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742281_1457 (size=12493) 2024-11-10T18:15:47,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742280_1456 (size=12493) 2024-11-10T18:15:47,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:47,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T18:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:47,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:47,075 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/ee69d028962646d381c6a48a0402d1fb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/ee69d028962646d381c6a48a0402d1fb 2024-11-10T18:15:47,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/62ff69f9064c48b2a35fc537843dee15 is 50, key is test_row_0/A:col10/1731262546297/Put/seqid=0 2024-11-10T18:15:47,082 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into ee69d028962646d381c6a48a0402d1fb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:47,082 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:47,082 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=13, startTime=1731262547042; duration=0sec 2024-11-10T18:15:47,082 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:47,082 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:47,082 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:47,083 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:47,083 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:47,083 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,083 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0fc0656368174ab5aa4449a1d694f17e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/954cb0ad30ab4ac68e1ea68f88ee3b42, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d5952e038f68445f90470bb0f835cfbe] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=35.8 K 2024-11-10T18:15:47,083 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fc0656368174ab5aa4449a1d694f17e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731262542777 2024-11-10T18:15:47,083 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 954cb0ad30ab4ac68e1ea68f88ee3b42, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731262543983 2024-11-10T18:15:47,084 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d5952e038f68445f90470bb0f835cfbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731262546182 2024-11-10T18:15:47,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742282_1458 (size=14541) 2024-11-10T18:15:47,089 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:47,089 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d3d1a0f80d6440b7a89742b7fa80390d is 50, key is test_row_0/C:col10/1731262546188/Put/seqid=0 2024-11-10T18:15:47,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742283_1459 (size=12493) 2024-11-10T18:15:47,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-10T18:15:47,135 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:47,135 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-10T18:15:47,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:47,137 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:47,138 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:47,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:47,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262607148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:47,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262607265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,277 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:47,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:47,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:47,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:47,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:47,474 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/84fab6e45c95414ba9fdcf1c9e2b7568 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/84fab6e45c95414ba9fdcf1c9e2b7568 2024-11-10T18:15:47,478 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into 84fab6e45c95414ba9fdcf1c9e2b7568(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:47,478 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:47,478 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=13, startTime=1731262547042; duration=0sec 2024-11-10T18:15:47,478 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:47,478 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:47,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/62ff69f9064c48b2a35fc537843dee15 2024-11-10T18:15:47,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/327bfab011c24428a41ca1825d47d8d1 is 50, key is test_row_0/B:col10/1731262546297/Put/seqid=0 2024-11-10T18:15:47,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742284_1460 (size=12151) 2024-11-10T18:15:47,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262607484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,506 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d3d1a0f80d6440b7a89742b7fa80390d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d3d1a0f80d6440b7a89742b7fa80390d 2024-11-10T18:15:47,509 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into d3d1a0f80d6440b7a89742b7fa80390d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:47,509 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:47,509 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=13, startTime=1731262547042; duration=0sec 2024-11-10T18:15:47,510 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:47,510 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:47,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262607515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,545 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262607535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:47,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:47,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:47,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:47,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:47,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262607804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:47,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:47,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/327bfab011c24428a41ca1825d47d8d1 2024-11-10T18:15:47,902 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/e693ab2015a2474cb6b9b6c842915ebb is 50, key is test_row_0/C:col10/1731262546297/Put/seqid=0 2024-11-10T18:15:47,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:47,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:47,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742285_1461 (size=12151) 2024-11-10T18:15:48,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:48,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/e693ab2015a2474cb6b9b6c842915ebb 2024-11-10T18:15:48,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/62ff69f9064c48b2a35fc537843dee15 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/62ff69f9064c48b2a35fc537843dee15 2024-11-10T18:15:48,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/62ff69f9064c48b2a35fc537843dee15, entries=200, sequenceid=203, filesize=14.2 K 2024-11-10T18:15:48,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/327bfab011c24428a41ca1825d47d8d1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/327bfab011c24428a41ca1825d47d8d1 2024-11-10T18:15:48,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/327bfab011c24428a41ca1825d47d8d1, entries=150, sequenceid=203, filesize=11.9 K 2024-11-10T18:15:48,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/e693ab2015a2474cb6b9b6c842915ebb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e693ab2015a2474cb6b9b6c842915ebb 2024-11-10T18:15:48,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e693ab2015a2474cb6b9b6c842915ebb, entries=150, sequenceid=203, filesize=11.9 K 2024-11-10T18:15:48,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 6367e59850ed14d0f0996a4816a8c486 in 1247ms, sequenceid=203, compaction requested=false 2024-11-10T18:15:48,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:48,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:48,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:48,328 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:48,329 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:48,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/a0675fd342cc4cdbac02f3dce948ba4c is 50, key is test_row_0/A:col10/1731262547109/Put/seqid=0 2024-11-10T18:15:48,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742286_1462 (size=12147) 2024-11-10T18:15:48,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262548385, max timestamp=1731262548429Num uniques 859; 2024-11-10T18:15:48,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:48,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262548385, max timestamp=1731262548429Num uniques 859; 2024-11-10T18:15:48,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:48,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262548385, max timestamp=1731262548429Num uniques 859; 2024-11-10T18:15:48,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:48,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262608546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262608555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262608560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,564 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262608675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262608675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/a0675fd342cc4cdbac02f3dce948ba4c 2024-11-10T18:15:48,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/8f3c932b38e94811bea8f00064f58643 is 50, key is test_row_0/B:col10/1731262547109/Put/seqid=0 2024-11-10T18:15:48,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742287_1463 (size=9757) 2024-11-10T18:15:48,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262608895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262608895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35138 deadline: 1731262608934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:48,946 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8290 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:48,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:48,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:48,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:48,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:49,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:49,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/8f3c932b38e94811bea8f00064f58643 2024-11-10T18:15:49,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/c58e99b68eea48a7bb8ef6d216de23b1 is 50, key is test_row_0/C:col10/1731262547109/Put/seqid=0 2024-11-10T18:15:49,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742288_1464 (size=9757) 2024-11-10T18:15:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:49,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262609214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:49,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:49,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262609214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:49,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:49,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:49,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/c58e99b68eea48a7bb8ef6d216de23b1 2024-11-10T18:15:49,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/a0675fd342cc4cdbac02f3dce948ba4c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/a0675fd342cc4cdbac02f3dce948ba4c 2024-11-10T18:15:49,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/a0675fd342cc4cdbac02f3dce948ba4c, entries=150, sequenceid=216, filesize=11.9 K 2024-11-10T18:15:49,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/8f3c932b38e94811bea8f00064f58643 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/8f3c932b38e94811bea8f00064f58643 2024-11-10T18:15:49,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/8f3c932b38e94811bea8f00064f58643, entries=100, sequenceid=216, filesize=9.5 K 2024-11-10T18:15:49,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/c58e99b68eea48a7bb8ef6d216de23b1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c58e99b68eea48a7bb8ef6d216de23b1 2024-11-10T18:15:49,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c58e99b68eea48a7bb8ef6d216de23b1, entries=100, sequenceid=216, filesize=9.5 K 2024-11-10T18:15:49,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6367e59850ed14d0f0996a4816a8c486 in 1245ms, sequenceid=216, compaction requested=true 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:49,573 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:49,573 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:49,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39181 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:49,574 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,574 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,574 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/ee69d028962646d381c6a48a0402d1fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/327bfab011c24428a41ca1825d47d8d1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/8f3c932b38e94811bea8f00064f58643] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=33.6 K 2024-11-10T18:15:49,574 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/84fab6e45c95414ba9fdcf1c9e2b7568, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/62ff69f9064c48b2a35fc537843dee15, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/a0675fd342cc4cdbac02f3dce948ba4c] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=38.3 K 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84fab6e45c95414ba9fdcf1c9e2b7568, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731262546182 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ee69d028962646d381c6a48a0402d1fb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731262546182 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 327bfab011c24428a41ca1825d47d8d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731262546297 2024-11-10T18:15:49,574 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62ff69f9064c48b2a35fc537843dee15, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731262546297 2024-11-10T18:15:49,575 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0675fd342cc4cdbac02f3dce948ba4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731262547109 2024-11-10T18:15:49,575 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f3c932b38e94811bea8f00064f58643, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731262547109 2024-11-10T18:15:49,586 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:49,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-10T18:15:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:49,586 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T18:15:49,587 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/a2f31db52a814395b3de17c21dc9a4a1 is 50, key is test_row_0/B:col10/1731262547109/Put/seqid=0 2024-11-10T18:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:49,588 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:49,588 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/f929e1edbdda43b9ac3f5f2967214a16 is 50, key is test_row_0/A:col10/1731262547109/Put/seqid=0 2024-11-10T18:15:49,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/4c15695765294e2ca6ce0e844df60b29 is 50, key is test_row_0/A:col10/1731262548558/Put/seqid=0 2024-11-10T18:15:49,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742290_1466 (size=12595) 2024-11-10T18:15:49,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742289_1465 (size=12595) 2024-11-10T18:15:49,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742291_1467 (size=12151) 2024-11-10T18:15:49,601 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/4c15695765294e2ca6ce0e844df60b29 2024-11-10T18:15:49,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/a6629fc035934a2c914284d8eaa5b20f is 50, key is test_row_0/B:col10/1731262548558/Put/seqid=0 2024-11-10T18:15:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742292_1468 (size=12151) 2024-11-10T18:15:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:49,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:49,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262609767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:49,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262609774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:49,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262609885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:49,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262609885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:49,896 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:49,998 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/f929e1edbdda43b9ac3f5f2967214a16 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/f929e1edbdda43b9ac3f5f2967214a16 2024-11-10T18:15:50,001 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/a2f31db52a814395b3de17c21dc9a4a1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a2f31db52a814395b3de17c21dc9a4a1 2024-11-10T18:15:50,003 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into f929e1edbdda43b9ac3f5f2967214a16(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:50,003 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:50,003 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=13, startTime=1731262549573; duration=0sec 2024-11-10T18:15:50,003 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:50,003 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:50,003 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:50,004 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:50,004 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:50,004 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:50,005 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d3d1a0f80d6440b7a89742b7fa80390d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e693ab2015a2474cb6b9b6c842915ebb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c58e99b68eea48a7bb8ef6d216de23b1] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=33.6 K 2024-11-10T18:15:50,005 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into a2f31db52a814395b3de17c21dc9a4a1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:50,005 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:50,005 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=13, startTime=1731262549573; duration=0sec 2024-11-10T18:15:50,005 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:50,005 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:50,005 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3d1a0f80d6440b7a89742b7fa80390d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731262546182 2024-11-10T18:15:50,005 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e693ab2015a2474cb6b9b6c842915ebb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731262546297 2024-11-10T18:15:50,005 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58e99b68eea48a7bb8ef6d216de23b1, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731262547109 2024-11-10T18:15:50,010 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/a6629fc035934a2c914284d8eaa5b20f 2024-11-10T18:15:50,013 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:50,013 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/212c87170ce146f19d03e989beedbe1c is 50, key is test_row_0/C:col10/1731262547109/Put/seqid=0 2024-11-10T18:15:50,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/6085a5b729f04b4c8ecf46ae6b15beb2 is 50, key is test_row_0/C:col10/1731262548558/Put/seqid=0 2024-11-10T18:15:50,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742293_1469 (size=12595) 2024-11-10T18:15:50,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742294_1470 (size=12151) 2024-11-10T18:15:50,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262610104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262610104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:50,422 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/212c87170ce146f19d03e989beedbe1c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/212c87170ce146f19d03e989beedbe1c 2024-11-10T18:15:50,426 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into 212c87170ce146f19d03e989beedbe1c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:50,426 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:50,426 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=13, startTime=1731262549573; duration=0sec 2024-11-10T18:15:50,426 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:50,426 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:50,428 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/6085a5b729f04b4c8ecf46ae6b15beb2 2024-11-10T18:15:50,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/4c15695765294e2ca6ce0e844df60b29 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4c15695765294e2ca6ce0e844df60b29 2024-11-10T18:15:50,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,434 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4c15695765294e2ca6ce0e844df60b29, entries=150, sequenceid=253, filesize=11.9 K 2024-11-10T18:15:50,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262610425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262610425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:50,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/a6629fc035934a2c914284d8eaa5b20f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a6629fc035934a2c914284d8eaa5b20f 2024-11-10T18:15:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,435 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:50,438 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a6629fc035934a2c914284d8eaa5b20f, entries=150, sequenceid=253, filesize=11.9 K 2024-11-10T18:15:50,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/6085a5b729f04b4c8ecf46ae6b15beb2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/6085a5b729f04b4c8ecf46ae6b15beb2 2024-11-10T18:15:50,441 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/6085a5b729f04b4c8ecf46ae6b15beb2, entries=150, sequenceid=253, filesize=11.9 K 2024-11-10T18:15:50,442 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6367e59850ed14d0f0996a4816a8c486 in 856ms, sequenceid=253, compaction requested=false 2024-11-10T18:15:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:50,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-10T18:15:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-10T18:15:50,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-10T18:15:50,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3050 sec 2024-11-10T18:15:50,446 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 3.3100 sec 2024-11-10T18:15:50,493 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T18:15:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:50,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:15:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:50,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:50,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/5f886b1dba1b4bc2bdcea3a39dca8fb3 is 50, key is test_row_0/A:col10/1731262549765/Put/seqid=0 2024-11-10T18:15:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742295_1471 (size=12301) 2024-11-10T18:15:50,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/5f886b1dba1b4bc2bdcea3a39dca8fb3 2024-11-10T18:15:50,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/e95400df86be4004b9f0e26e9c5ffbe9 is 50, key is test_row_0/B:col10/1731262549765/Put/seqid=0 2024-11-10T18:15:50,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742296_1472 (size=12301) 2024-11-10T18:15:50,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/e95400df86be4004b9f0e26e9c5ffbe9 2024-11-10T18:15:50,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/2e157cdd5be84192af8a90b208add143 is 50, key is test_row_0/C:col10/1731262549765/Put/seqid=0 2024-11-10T18:15:50,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742297_1473 (size=12301) 2024-11-10T18:15:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262550666, max timestamp=1731262550742Num uniques 859; 2024-11-10T18:15:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262550666, max timestamp=1731262550742Num uniques 859; 2024-11-10T18:15:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262550666, max timestamp=1731262550742Num uniques 859; 2024-11-10T18:15:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:50,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262610884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:50,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262610945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262610945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:50,951 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262611004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,029 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/2e157cdd5be84192af8a90b208add143 2024-11-10T18:15:51,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/5f886b1dba1b4bc2bdcea3a39dca8fb3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5f886b1dba1b4bc2bdcea3a39dca8fb3 2024-11-10T18:15:51,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5f886b1dba1b4bc2bdcea3a39dca8fb3, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T18:15:51,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/e95400df86be4004b9f0e26e9c5ffbe9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/e95400df86be4004b9f0e26e9c5ffbe9 2024-11-10T18:15:51,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/e95400df86be4004b9f0e26e9c5ffbe9, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T18:15:51,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/2e157cdd5be84192af8a90b208add143 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2e157cdd5be84192af8a90b208add143 2024-11-10T18:15:51,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2e157cdd5be84192af8a90b208add143, entries=150, sequenceid=267, filesize=12.0 K 2024-11-10T18:15:51,043 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6367e59850ed14d0f0996a4816a8c486 in 465ms, sequenceid=267, compaction requested=true 2024-11-10T18:15:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:51,043 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:51,043 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:51,043 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:51,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:51,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:51,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:51,044 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:51,044 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:51,044 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,044 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a2f31db52a814395b3de17c21dc9a4a1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a6629fc035934a2c914284d8eaa5b20f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/e95400df86be4004b9f0e26e9c5ffbe9] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=36.2 K 2024-11-10T18:15:51,045 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:51,045 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f31db52a814395b3de17c21dc9a4a1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731262546348 2024-11-10T18:15:51,045 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:51,045 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,045 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/f929e1edbdda43b9ac3f5f2967214a16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4c15695765294e2ca6ce0e844df60b29, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5f886b1dba1b4bc2bdcea3a39dca8fb3] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=36.2 K 2024-11-10T18:15:51,045 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a6629fc035934a2c914284d8eaa5b20f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731262548537 2024-11-10T18:15:51,045 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f929e1edbdda43b9ac3f5f2967214a16, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731262546348 2024-11-10T18:15:51,045 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting e95400df86be4004b9f0e26e9c5ffbe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731262549765 2024-11-10T18:15:51,046 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c15695765294e2ca6ce0e844df60b29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731262548537 2024-11-10T18:15:51,046 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f886b1dba1b4bc2bdcea3a39dca8fb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731262549765 2024-11-10T18:15:51,052 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:51,053 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:51,053 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/2ffa2490ca04438c90fe7312dfa70807 is 50, key is test_row_0/B:col10/1731262549765/Put/seqid=0 2024-11-10T18:15:51,053 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/fb4273248ff44ad6a0b57bec50b8bac5 is 50, key is test_row_0/A:col10/1731262549765/Put/seqid=0 2024-11-10T18:15:51,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742298_1474 (size=12847) 2024-11-10T18:15:51,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742299_1475 (size=12847) 2024-11-10T18:15:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:51,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T18:15:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:51,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:51,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/da1d0430161644278f6e410c2fd536e1 is 50, key is test_row_0/A:col10/1731262550863/Put/seqid=0 2024-11-10T18:15:51,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742300_1476 (size=14741) 2024-11-10T18:15:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262611292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-10T18:15:51,305 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:51,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-10T18:15:51,307 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T18:15:51,308 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:51,308 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:51,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262611405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,409 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T18:15:51,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:51,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:51,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,462 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/2ffa2490ca04438c90fe7312dfa70807 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2ffa2490ca04438c90fe7312dfa70807 2024-11-10T18:15:51,465 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/fb4273248ff44ad6a0b57bec50b8bac5 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/fb4273248ff44ad6a0b57bec50b8bac5 2024-11-10T18:15:51,466 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into 2ffa2490ca04438c90fe7312dfa70807(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:51,466 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:51,466 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=13, startTime=1731262551043; duration=0sec 2024-11-10T18:15:51,466 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:51,466 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:51,466 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:51,467 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:51,467 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:51,467 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,467 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/212c87170ce146f19d03e989beedbe1c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/6085a5b729f04b4c8ecf46ae6b15beb2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2e157cdd5be84192af8a90b208add143] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=36.2 K 2024-11-10T18:15:51,467 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 212c87170ce146f19d03e989beedbe1c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1731262546348 2024-11-10T18:15:51,468 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6085a5b729f04b4c8ecf46ae6b15beb2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731262548537 2024-11-10T18:15:51,468 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e157cdd5be84192af8a90b208add143, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731262549765 2024-11-10T18:15:51,468 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into fb4273248ff44ad6a0b57bec50b8bac5(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:51,469 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:51,469 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=13, startTime=1731262551043; duration=0sec 2024-11-10T18:15:51,469 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:51,469 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:51,475 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:51,475 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/7fc85a8908e04433bfa1608bb2a093d0 is 50, key is test_row_0/C:col10/1731262549765/Put/seqid=0 2024-11-10T18:15:51,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742301_1477 (size=12847) 2024-11-10T18:15:51,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:51,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262611615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T18:15:51,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/da1d0430161644278f6e410c2fd536e1 2024-11-10T18:15:51,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/c688689f32754185b0995cf16953a785 is 50, key is test_row_0/B:col10/1731262550863/Put/seqid=0 2024-11-10T18:15:51,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742302_1478 (size=12301) 2024-11-10T18:15:51,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,883 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/7fc85a8908e04433bfa1608bb2a093d0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7fc85a8908e04433bfa1608bb2a093d0 2024-11-10T18:15:51,887 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into 7fc85a8908e04433bfa1608bb2a093d0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:51,887 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:51,887 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=13, startTime=1731262551044; duration=0sec 2024-11-10T18:15:51,887 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:51,887 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:51,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:51,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:51,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262611924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,928 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T18:15:51,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262611965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:51,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262611965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:51,974 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:51,975 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:51,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:51,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:52,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/c688689f32754185b0995cf16953a785 2024-11-10T18:15:52,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/53d928da81684aa083990c0d548593b7 is 50, key is test_row_0/C:col10/1731262550863/Put/seqid=0 2024-11-10T18:15:52,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742303_1479 (size=12301) 2024-11-10T18:15:52,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:52,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:52,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:52,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262612434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:52,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T18:15:52,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/53d928da81684aa083990c0d548593b7 2024-11-10T18:15:52,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/da1d0430161644278f6e410c2fd536e1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/da1d0430161644278f6e410c2fd536e1 2024-11-10T18:15:52,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/da1d0430161644278f6e410c2fd536e1, entries=200, sequenceid=303, filesize=14.4 K 2024-11-10T18:15:52,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/c688689f32754185b0995cf16953a785 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/c688689f32754185b0995cf16953a785 2024-11-10T18:15:52,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/c688689f32754185b0995cf16953a785, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T18:15:52,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/53d928da81684aa083990c0d548593b7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/53d928da81684aa083990c0d548593b7 2024-11-10T18:15:52,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/53d928da81684aa083990c0d548593b7, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T18:15:52,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6367e59850ed14d0f0996a4816a8c486 in 1247ms, sequenceid=303, compaction requested=false 2024-11-10T18:15:52,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:52,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:52,531 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:52,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:52,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/5de88ab28918402baa6728fa0e98f21f is 50, key is test_row_0/A:col10/1731262551262/Put/seqid=0 2024-11-10T18:15:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742304_1480 (size=12301) 2024-11-10T18:15:52,940 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/5de88ab28918402baa6728fa0e98f21f 2024-11-10T18:15:52,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/3c2cec8c750b4083b63b11db5cb5774d is 50, key is test_row_0/B:col10/1731262551262/Put/seqid=0 2024-11-10T18:15:52,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742305_1481 (size=12301) 2024-11-10T18:15:53,351 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/3c2cec8c750b4083b63b11db5cb5774d 2024-11-10T18:15:53,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/f2e92c179db6421cab142c7945a44e32 is 50, key is test_row_0/C:col10/1731262551262/Put/seqid=0 2024-11-10T18:15:53,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742306_1482 (size=12301) 2024-11-10T18:15:53,378 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/f2e92c179db6421cab142c7945a44e32 2024-11-10T18:15:53,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/5de88ab28918402baa6728fa0e98f21f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5de88ab28918402baa6728fa0e98f21f 2024-11-10T18:15:53,385 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5de88ab28918402baa6728fa0e98f21f, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T18:15:53,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/3c2cec8c750b4083b63b11db5cb5774d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3c2cec8c750b4083b63b11db5cb5774d 2024-11-10T18:15:53,390 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3c2cec8c750b4083b63b11db5cb5774d, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T18:15:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/f2e92c179db6421cab142c7945a44e32 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f2e92c179db6421cab142c7945a44e32 2024-11-10T18:15:53,394 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f2e92c179db6421cab142c7945a44e32, entries=150, sequenceid=316, filesize=12.0 K 2024-11-10T18:15:53,395 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=0 B/0 for 6367e59850ed14d0f0996a4816a8c486 in 864ms, sequenceid=316, compaction requested=true 2024-11-10T18:15:53,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:53,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-10T18:15:53,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-10T18:15:53,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-10T18:15:53,400 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0890 sec 2024-11-10T18:15:53,402 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.0950 sec 2024-11-10T18:15:53,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-10T18:15:53,455 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:53,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:53,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-10T18:15:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:53,459 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:53,460 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:53,460 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:53,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:53,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:53,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:53,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/9a0c448e6f934ef4a430e0fba6793d53 is 50, key is test_row_0/A:col10/1731262553477/Put/seqid=0 2024-11-10T18:15:53,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742307_1483 (size=12297) 2024-11-10T18:15:53,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:53,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262553575, max timestamp=1731262553614Num uniques 859; 2024-11-10T18:15:53,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:53,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262553575, max timestamp=1731262553614Num uniques 859; 2024-11-10T18:15:53,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:53,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262553575, max timestamp=1731262553614Num uniques 859; 2024-11-10T18:15:53,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:15:53,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:53,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262613751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:53,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:53,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:53,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:53,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:53,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:53,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262613855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:53,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:53,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/9a0c448e6f934ef4a430e0fba6793d53 2024-11-10T18:15:53,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/43e5462828ca4ebdb730da3f58154f28 is 50, key is test_row_0/B:col10/1731262553477/Put/seqid=0 2024-11-10T18:15:53,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742308_1484 (size=9857) 2024-11-10T18:15:53,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:53,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:53,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:53,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:53,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262613984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:53,996 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:54,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262613994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:54,008 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:54,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262614074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:54,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:54,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:54,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/43e5462828ca4ebdb730da3f58154f28 2024-11-10T18:15:54,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/ff560f81cb6e4ca18af20d2f164d1d1d is 50, key is test_row_0/C:col10/1731262553477/Put/seqid=0 2024-11-10T18:15:54,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742309_1485 (size=9857) 2024-11-10T18:15:54,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:54,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:54,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262614394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:54,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:54,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:54,706 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=326 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/ff560f81cb6e4ca18af20d2f164d1d1d 2024-11-10T18:15:54,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/9a0c448e6f934ef4a430e0fba6793d53 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9a0c448e6f934ef4a430e0fba6793d53 2024-11-10T18:15:54,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9a0c448e6f934ef4a430e0fba6793d53, entries=150, sequenceid=326, filesize=12.0 K 2024-11-10T18:15:54,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/43e5462828ca4ebdb730da3f58154f28 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/43e5462828ca4ebdb730da3f58154f28 2024-11-10T18:15:54,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/43e5462828ca4ebdb730da3f58154f28, entries=100, sequenceid=326, filesize=9.6 K 2024-11-10T18:15:54,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/ff560f81cb6e4ca18af20d2f164d1d1d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/ff560f81cb6e4ca18af20d2f164d1d1d 2024-11-10T18:15:54,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/ff560f81cb6e4ca18af20d2f164d1d1d, entries=100, sequenceid=326, filesize=9.6 K 2024-11-10T18:15:54,724 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6367e59850ed14d0f0996a4816a8c486 in 1246ms, sequenceid=326, compaction requested=true 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:54,724 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:54,724 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:54,725 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:54,725 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52186 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:54,726 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:54,726 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:54,726 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,726 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,726 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2ffa2490ca04438c90fe7312dfa70807, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/c688689f32754185b0995cf16953a785, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3c2cec8c750b4083b63b11db5cb5774d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/43e5462828ca4ebdb730da3f58154f28] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=46.2 K 2024-11-10T18:15:54,726 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/fb4273248ff44ad6a0b57bec50b8bac5, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/da1d0430161644278f6e410c2fd536e1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5de88ab28918402baa6728fa0e98f21f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9a0c448e6f934ef4a430e0fba6793d53] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=51.0 K 2024-11-10T18:15:54,726 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ffa2490ca04438c90fe7312dfa70807, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731262549765 2024-11-10T18:15:54,726 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb4273248ff44ad6a0b57bec50b8bac5, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731262549765 2024-11-10T18:15:54,726 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting da1d0430161644278f6e410c2fd536e1, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731262550863 2024-11-10T18:15:54,726 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c688689f32754185b0995cf16953a785, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731262550863 2024-11-10T18:15:54,727 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5de88ab28918402baa6728fa0e98f21f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731262551262 2024-11-10T18:15:54,727 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c2cec8c750b4083b63b11db5cb5774d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731262551262 2024-11-10T18:15:54,727 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 43e5462828ca4ebdb730da3f58154f28, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731262553469 2024-11-10T18:15:54,727 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a0c448e6f934ef4a430e0fba6793d53, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731262553469 2024-11-10T18:15:54,734 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:54,734 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6849db496bbf44c998dd3fdb26e975fd is 50, key is test_row_0/B:col10/1731262553477/Put/seqid=0 2024-11-10T18:15:54,738 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#415 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:54,738 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/512bbdbe6d444926825bab72b64051ff is 50, key is test_row_0/A:col10/1731262553477/Put/seqid=0 2024-11-10T18:15:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742310_1486 (size=12983) 2024-11-10T18:15:54,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742311_1487 (size=12983) 2024-11-10T18:15:54,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-10T18:15:54,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:54,836 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T18:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:54,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/1ca0d5a20ccd483a8c967ccfbee23d9d is 50, key is test_row_0/A:col10/1731262553718/Put/seqid=0 2024-11-10T18:15:54,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742312_1488 (size=12301) 2024-11-10T18:15:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:54,972 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:54,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262614968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:54,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:55,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:55,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262615074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:55,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:55,144 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6849db496bbf44c998dd3fdb26e975fd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6849db496bbf44c998dd3fdb26e975fd 2024-11-10T18:15:55,146 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/512bbdbe6d444926825bab72b64051ff as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/512bbdbe6d444926825bab72b64051ff 2024-11-10T18:15:55,148 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into 6849db496bbf44c998dd3fdb26e975fd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:55,148 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:55,148 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=12, startTime=1731262554724; duration=0sec 2024-11-10T18:15:55,148 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:55,148 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:55,149 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47306 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:55,150 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into 512bbdbe6d444926825bab72b64051ff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:55,150 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:55,150 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=12, startTime=1731262554724; duration=0sec 2024-11-10T18:15:55,150 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7fc85a8908e04433bfa1608bb2a093d0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/53d928da81684aa083990c0d548593b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f2e92c179db6421cab142c7945a44e32, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/ff560f81cb6e4ca18af20d2f164d1d1d] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=46.2 K 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fc85a8908e04433bfa1608bb2a093d0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1731262549765 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d928da81684aa083990c0d548593b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731262550863 2024-11-10T18:15:55,150 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f2e92c179db6421cab142c7945a44e32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731262551262 2024-11-10T18:15:55,151 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ff560f81cb6e4ca18af20d2f164d1d1d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731262553469 2024-11-10T18:15:55,166 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:55,167 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/3bfca628a6474d2aba8254a26e169689 is 50, key is test_row_0/C:col10/1731262553477/Put/seqid=0 2024-11-10T18:15:55,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742313_1489 (size=12983) 2024-11-10T18:15:55,244 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/1ca0d5a20ccd483a8c967ccfbee23d9d 2024-11-10T18:15:55,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/06e7cf85d7b245a88b75e996ca718925 is 50, key is test_row_0/B:col10/1731262553718/Put/seqid=0 2024-11-10T18:15:55,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742314_1490 (size=12301) 2024-11-10T18:15:55,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:55,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262615285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:55,582 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/3bfca628a6474d2aba8254a26e169689 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/3bfca628a6474d2aba8254a26e169689 2024-11-10T18:15:55,585 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into 3bfca628a6474d2aba8254a26e169689(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:55,586 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:55,586 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=12, startTime=1731262554724; duration=0sec 2024-11-10T18:15:55,586 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:55,586 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:55,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:55,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:55,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262615605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:55,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:55,655 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/06e7cf85d7b245a88b75e996ca718925 2024-11-10T18:15:55,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/e229978dde36478f8806165c0511fb36 is 50, key is test_row_0/C:col10/1731262553718/Put/seqid=0 2024-11-10T18:15:55,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742315_1491 (size=12301) 2024-11-10T18:15:56,068 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/e229978dde36478f8806165c0511fb36 2024-11-10T18:15:56,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/1ca0d5a20ccd483a8c967ccfbee23d9d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/1ca0d5a20ccd483a8c967ccfbee23d9d 2024-11-10T18:15:56,075 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/1ca0d5a20ccd483a8c967ccfbee23d9d, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T18:15:56,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/06e7cf85d7b245a88b75e996ca718925 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/06e7cf85d7b245a88b75e996ca718925 2024-11-10T18:15:56,079 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/06e7cf85d7b245a88b75e996ca718925, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T18:15:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/e229978dde36478f8806165c0511fb36 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e229978dde36478f8806165c0511fb36 2024-11-10T18:15:56,083 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e229978dde36478f8806165c0511fb36, entries=150, sequenceid=362, filesize=12.0 K 2024-11-10T18:15:56,083 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6367e59850ed14d0f0996a4816a8c486 in 1248ms, sequenceid=362, compaction requested=false 2024-11-10T18:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:56,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-10T18:15:56,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-10T18:15:56,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-10T18:15:56,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6250 sec 2024-11-10T18:15:56,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.6320 sec 2024-11-10T18:15:56,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:56,130 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-10T18:15:56,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:56,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:56,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:56,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:56,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:56,130 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:56,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/b81a25b53f614f9889c2a6778ef212cd is 50, key is test_row_0/A:col10/1731262554938/Put/seqid=0 2024-11-10T18:15:56,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742316_1492 (size=12301) 2024-11-10T18:15:56,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/b81a25b53f614f9889c2a6778ef212cd 2024-11-10T18:15:56,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/14d8cb07c52848de9b122c866a024065 is 50, key is test_row_0/B:col10/1731262554938/Put/seqid=0 2024-11-10T18:15:56,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742317_1493 (size=12301) 2024-11-10T18:15:56,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262556211, max timestamp=1731262556263Num uniques 859; 2024-11-10T18:15:56,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262556211, max timestamp=1731262556263Num uniques 859; 2024-11-10T18:15:56,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262556211, max timestamp=1731262556263Num uniques 859; 2024-11-10T18:15:56,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:56,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:56,266 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:15:56,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:56,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262616410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:56,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:56,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262616525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:56,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:56,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/14d8cb07c52848de9b122c866a024065 2024-11-10T18:15:56,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/7d2af37d2d974f088ea24294717dfbb8 is 50, key is test_row_0/C:col10/1731262554938/Put/seqid=0 2024-11-10T18:15:56,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742318_1494 (size=12301) 2024-11-10T18:15:56,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262616744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:56,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:56,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/7d2af37d2d974f088ea24294717dfbb8 2024-11-10T18:15:56,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/b81a25b53f614f9889c2a6778ef212cd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/b81a25b53f614f9889c2a6778ef212cd 2024-11-10T18:15:56,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/b81a25b53f614f9889c2a6778ef212cd, entries=150, sequenceid=376, filesize=12.0 K 2024-11-10T18:15:56,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/14d8cb07c52848de9b122c866a024065 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/14d8cb07c52848de9b122c866a024065 2024-11-10T18:15:56,979 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/14d8cb07c52848de9b122c866a024065, entries=150, sequenceid=376, filesize=12.0 K 2024-11-10T18:15:56,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/7d2af37d2d974f088ea24294717dfbb8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7d2af37d2d974f088ea24294717dfbb8 2024-11-10T18:15:56,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7d2af37d2d974f088ea24294717dfbb8, entries=150, sequenceid=376, filesize=12.0 K 2024-11-10T18:15:56,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6367e59850ed14d0f0996a4816a8c486 in 853ms, sequenceid=376, compaction requested=true 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:15:56,984 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6367e59850ed14d0f0996a4816a8c486:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:15:56,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:56,984 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/A is initiating minor compaction (all files) 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/B is initiating minor compaction (all files) 2024-11-10T18:15:56,985 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/A in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:56,985 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/B in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:56,985 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/512bbdbe6d444926825bab72b64051ff, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/1ca0d5a20ccd483a8c967ccfbee23d9d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/b81a25b53f614f9889c2a6778ef212cd] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=36.7 K 2024-11-10T18:15:56,985 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6849db496bbf44c998dd3fdb26e975fd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/06e7cf85d7b245a88b75e996ca718925, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/14d8cb07c52848de9b122c866a024065] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=36.7 K 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6849db496bbf44c998dd3fdb26e975fd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731262551290 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512bbdbe6d444926825bab72b64051ff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731262551290 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca0d5a20ccd483a8c967ccfbee23d9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731262553718 2024-11-10T18:15:56,985 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 06e7cf85d7b245a88b75e996ca718925, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731262553718 2024-11-10T18:15:56,986 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting b81a25b53f614f9889c2a6778ef212cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731262554938 2024-11-10T18:15:56,986 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 14d8cb07c52848de9b122c866a024065, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731262554938 2024-11-10T18:15:56,991 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#A#compaction#423 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:56,991 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/d1d6945ef8ca49f6977d5144852f50cb is 50, key is test_row_0/A:col10/1731262554938/Put/seqid=0 2024-11-10T18:15:56,994 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#B#compaction#424 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:56,995 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/3d1338c039c24d379c0ea99189db6712 is 50, key is test_row_0/B:col10/1731262554938/Put/seqid=0 2024-11-10T18:15:57,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742319_1495 (size=13085) 2024-11-10T18:15:57,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742320_1496 (size=13085) 2024-11-10T18:15:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:57,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-10T18:15:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:57,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:15:57,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/951d46dee26447ca81b32290da3ac53b is 50, key is test_row_0/A:col10/1731262556397/Put/seqid=0 2024-11-10T18:15:57,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742321_1497 (size=14741) 2024-11-10T18:15:57,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262617126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262617128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262617234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262617255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,407 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/d1d6945ef8ca49f6977d5144852f50cb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/d1d6945ef8ca49f6977d5144852f50cb 2024-11-10T18:15:57,412 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/A of 6367e59850ed14d0f0996a4816a8c486 into d1d6945ef8ca49f6977d5144852f50cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:57,412 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:57,412 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/A, priority=13, startTime=1731262556984; duration=0sec 2024-11-10T18:15:57,412 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:15:57,412 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:A 2024-11-10T18:15:57,412 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:15:57,413 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:15:57,413 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/3d1338c039c24d379c0ea99189db6712 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3d1338c039c24d379c0ea99189db6712 2024-11-10T18:15:57,414 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 6367e59850ed14d0f0996a4816a8c486/C is initiating minor compaction (all files) 2024-11-10T18:15:57,414 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6367e59850ed14d0f0996a4816a8c486/C in TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:57,414 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/3bfca628a6474d2aba8254a26e169689, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e229978dde36478f8806165c0511fb36, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7d2af37d2d974f088ea24294717dfbb8] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp, totalSize=36.7 K 2024-11-10T18:15:57,414 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bfca628a6474d2aba8254a26e169689, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=326, earliestPutTs=1731262551290 2024-11-10T18:15:57,415 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e229978dde36478f8806165c0511fb36, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1731262553718 2024-11-10T18:15:57,415 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d2af37d2d974f088ea24294717dfbb8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1731262554938 2024-11-10T18:15:57,418 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/B of 6367e59850ed14d0f0996a4816a8c486 into 3d1338c039c24d379c0ea99189db6712(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:57,418 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:57,418 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/B, priority=13, startTime=1731262556984; duration=0sec 2024-11-10T18:15:57,418 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:57,418 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:B 2024-11-10T18:15:57,423 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6367e59850ed14d0f0996a4816a8c486#C#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:15:57,423 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/0deee92da284424d9ac8111794d1cb66 is 50, key is test_row_0/C:col10/1731262554938/Put/seqid=0 2024-11-10T18:15:57,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742322_1498 (size=13085) 2024-11-10T18:15:57,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262617454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262617475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/951d46dee26447ca81b32290da3ac53b 2024-11-10T18:15:57,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6f971f2bb608458590ed2971a059c866 is 50, key is test_row_0/B:col10/1731262556397/Put/seqid=0 2024-11-10T18:15:57,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742323_1499 (size=12301) 2024-11-10T18:15:57,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-10T18:15:57,635 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:15:57,636 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-10T18:15:57,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T18:15:57,637 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:15:57,638 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:15:57,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:15:57,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T18:15:57,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262617774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T18:15:57,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:57,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:57,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:57,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:57,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:57,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:57,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:57,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262617795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:57,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:57,833 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/0deee92da284424d9ac8111794d1cb66 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0deee92da284424d9ac8111794d1cb66 2024-11-10T18:15:57,836 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6367e59850ed14d0f0996a4816a8c486/C of 6367e59850ed14d0f0996a4816a8c486 into 0deee92da284424d9ac8111794d1cb66(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:15:57,836 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:57,836 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., storeName=6367e59850ed14d0f0996a4816a8c486/C, priority=13, startTime=1731262556984; duration=0sec 2024-11-10T18:15:57,837 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:15:57,837 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6367e59850ed14d0f0996a4816a8c486:C 2024-11-10T18:15:57,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6f971f2bb608458590ed2971a059c866 2024-11-10T18:15:57,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d8f56ef86fc640d9bb1653942bc0473e is 50, key is test_row_0/C:col10/1731262556397/Put/seqid=0 2024-11-10T18:15:57,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742324_1500 (size=12301) 2024-11-10T18:15:57,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T18:15:57,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:57,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:57,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:57,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:57,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:57,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T18:15:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35230 deadline: 1731262618025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:58,030 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8256 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:58,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:58,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35152 deadline: 1731262618045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:58,055 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 6367e59850ed14d0f0996a4816a8c486 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:15:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:58,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T18:15:58,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:58,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:58,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:58,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:58,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T18:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:58,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:58,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:58,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:58,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:15:58,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T18:15:58,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:58,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35182 deadline: 1731262618285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,298 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:58,311 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d8f56ef86fc640d9bb1653942bc0473e 2024-11-10T18:15:58,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/951d46dee26447ca81b32290da3ac53b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/951d46dee26447ca81b32290da3ac53b 2024-11-10T18:15:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:15:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35192 deadline: 1731262618314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 2024-11-10T18:15:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 , the old value is region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:15:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., hostname=691ea56905d2,32909,1731262402364, seqNum=2 because the exception is null or not the one we care about 2024-11-10T18:15:58,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/951d46dee26447ca81b32290da3ac53b, entries=200, sequenceid=413, filesize=14.4 K 2024-11-10T18:15:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/6f971f2bb608458590ed2971a059c866 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6f971f2bb608458590ed2971a059c866 2024-11-10T18:15:58,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6f971f2bb608458590ed2971a059c866, entries=150, sequenceid=413, filesize=12.0 K 2024-11-10T18:15:58,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/d8f56ef86fc640d9bb1653942bc0473e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d8f56ef86fc640d9bb1653942bc0473e 2024-11-10T18:15:58,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d8f56ef86fc640d9bb1653942bc0473e, entries=150, sequenceid=413, filesize=12.0 K 2024-11-10T18:15:58,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6367e59850ed14d0f0996a4816a8c486 in 1253ms, sequenceid=413, compaction requested=false 2024-11-10T18:15:58,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:58,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-10T18:15:58,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:58,402 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:15:58,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:15:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:15:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:15:58,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:15:58,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/15ae235fe105403ba22283ac73ed09ac is 50, key is test_row_0/A:col10/1731262557100/Put/seqid=0 2024-11-10T18:15:58,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742325_1501 (size=12301) 2024-11-10T18:15:58,683 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-10T18:15:58,683 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:58,683 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,683 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:58,685 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-10T18:15:58,685 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:58,685 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,685 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:58,686 INFO [Thread-1940 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1940. 2024-11-10T18:15:58,686 DEBUG [Thread-1940 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:58,686 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,686 DEBUG [Thread-1940 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,686 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:58,690 INFO [Thread-1938 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1938. 2024-11-10T18:15:58,690 DEBUG [Thread-1938 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:58,690 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,690 DEBUG [Thread-1938 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,690 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:58,691 INFO [Thread-1942 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1942. 2024-11-10T18:15:58,691 DEBUG [Thread-1942 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:58,691 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,691 DEBUG [Thread-1942 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,691 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T18:15:58,811 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/15ae235fe105403ba22283ac73ed09ac 2024-11-10T18:15:58,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/2954cada57b44242a08e2b9ea44246d0 is 50, key is test_row_0/B:col10/1731262557100/Put/seqid=0 2024-11-10T18:15:58,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742326_1502 (size=12301) 2024-11-10T18:15:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:15:58,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. as already flushing 2024-11-10T18:15:58,966 INFO [Thread-1923 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1923. 2024-11-10T18:15:58,966 DEBUG [Thread-1923 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:58,966 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,966 DEBUG [Thread-1923 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:58,966 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:59,221 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/2954cada57b44242a08e2b9ea44246d0 2024-11-10T18:15:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/54064386f65c417681be5e5dfe0d9674 is 50, key is test_row_0/C:col10/1731262557100/Put/seqid=0 2024-11-10T18:15:59,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742327_1503 (size=12301) 2024-11-10T18:15:59,306 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-11-10T18:15:59,306 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:59,306 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:59,307 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:59,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:59,325 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-10T18:15:59,326 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:15:59,326 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:59,326 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:15:59,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:15:59,630 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/54064386f65c417681be5e5dfe0d9674 2024-11-10T18:15:59,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/15ae235fe105403ba22283ac73ed09ac as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/15ae235fe105403ba22283ac73ed09ac 2024-11-10T18:15:59,636 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/15ae235fe105403ba22283ac73ed09ac, entries=150, sequenceid=426, filesize=12.0 K 2024-11-10T18:15:59,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/2954cada57b44242a08e2b9ea44246d0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2954cada57b44242a08e2b9ea44246d0 2024-11-10T18:15:59,639 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2954cada57b44242a08e2b9ea44246d0, entries=150, sequenceid=426, filesize=12.0 K 2024-11-10T18:15:59,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/54064386f65c417681be5e5dfe0d9674 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/54064386f65c417681be5e5dfe0d9674 2024-11-10T18:15:59,642 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/54064386f65c417681be5e5dfe0d9674, entries=150, sequenceid=426, filesize=12.0 K 2024-11-10T18:15:59,643 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=20.13 KB/20610 for 6367e59850ed14d0f0996a4816a8c486 in 1241ms, sequenceid=426, compaction requested=true 2024-11-10T18:15:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6367e59850ed14d0f0996a4816a8c486: 2024-11-10T18:15:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:15:59,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-10T18:15:59,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-10T18:15:59,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-10T18:15:59,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0060 sec 2024-11-10T18:15:59,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.0100 sec 2024-11-10T18:15:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-10T18:15:59,785 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:08,096 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-11-10T18:16:08,096 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:08,096 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:08,096 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:08,096 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:08,165 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-11-10T18:16:08,166 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:08,166 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:08,166 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:08,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 191 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3167 rows 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1057 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3168 rows 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3179 rows 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1061 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3181 rows 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1058 2024-11-10T18:16:08,166 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3172 rows 2024-11-10T18:16:08,166 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:16:08,166 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:16:08,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:08,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:08,167 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:08,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T18:16:08,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:08,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T18:16:08,170 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262568169"}]},"ts":"1731262568169"} 2024-11-10T18:16:08,171 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T18:16:08,171 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T18:16:08,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:16:08,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, UNASSIGN}] 2024-11-10T18:16:08,173 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, UNASSIGN 2024-11-10T18:16:08,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=6367e59850ed14d0f0996a4816a8c486, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:08,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=131, ppid=130, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, UNASSIGN because future has completed 2024-11-10T18:16:08,175 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:16:08,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:16:08,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T18:16:08,326 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(122): Close 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:16:08,326 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1722): Closing 6367e59850ed14d0f0996a4816a8c486, disabling compactions & flushes 2024-11-10T18:16:08,327 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. after waiting 0 ms 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:16:08,327 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(2902): Flushing 6367e59850ed14d0f0996a4816a8c486 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=A 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=B 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6367e59850ed14d0f0996a4816a8c486, store=C 2024-11-10T18:16:08,327 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:08,330 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/4493d4becef64ea6baab7f52cc420408 is 50, key is test_row_0/A:col10/1731262568095/Put/seqid=0 2024-11-10T18:16:08,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742328_1504 (size=12301) 2024-11-10T18:16:08,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T18:16:08,734 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/4493d4becef64ea6baab7f52cc420408 2024-11-10T18:16:08,739 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/26a4449e8dc7410a94be7e126f0c56f0 is 50, key is test_row_0/B:col10/1731262568095/Put/seqid=0 2024-11-10T18:16:08,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742329_1505 (size=12301) 2024-11-10T18:16:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T18:16:09,143 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/26a4449e8dc7410a94be7e126f0c56f0 2024-11-10T18:16:09,148 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/a845467c7c1d4358b4bfba69c6ead282 is 50, key is test_row_0/C:col10/1731262568095/Put/seqid=0 2024-11-10T18:16:09,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742330_1506 (size=12301) 2024-11-10T18:16:09,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T18:16:09,551 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/a845467c7c1d4358b4bfba69c6ead282 2024-11-10T18:16:09,555 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/A/4493d4becef64ea6baab7f52cc420408 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4493d4becef64ea6baab7f52cc420408 2024-11-10T18:16:09,557 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4493d4becef64ea6baab7f52cc420408, entries=150, sequenceid=434, filesize=12.0 K 2024-11-10T18:16:09,558 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/B/26a4449e8dc7410a94be7e126f0c56f0 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/26a4449e8dc7410a94be7e126f0c56f0 2024-11-10T18:16:09,561 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/26a4449e8dc7410a94be7e126f0c56f0, entries=150, sequenceid=434, filesize=12.0 K 2024-11-10T18:16:09,561 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/.tmp/C/a845467c7c1d4358b4bfba69c6ead282 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a845467c7c1d4358b4bfba69c6ead282 2024-11-10T18:16:09,564 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a845467c7c1d4358b4bfba69c6ead282, entries=150, sequenceid=434, filesize=12.0 K 2024-11-10T18:16:09,565 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6367e59850ed14d0f0996a4816a8c486 in 1238ms, sequenceid=434, compaction requested=true 2024-11-10T18:16:09,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/7e76feae7e2c4b96be7204ecdf47f673, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/def3c05868954003bb7f413908f020c2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e709000c94ac47bba8849f74ea9c615c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e0d9daee4965492a88737a1085fbc13f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/cdf73db9b67b45fc91f020ffa755ff2b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8bcea8cb6d7d4734aff835f32bccc066, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/6e0718ee4bad4d3789c4d0b87e0ff4bf, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9e25497d425e451f97bc00433c65f5e7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e4c775a661ec4426a79cf1eee3c7f06c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8c51115b8508444cbb010d10608ee237, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/84fab6e45c95414ba9fdcf1c9e2b7568, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/62ff69f9064c48b2a35fc537843dee15, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/f929e1edbdda43b9ac3f5f2967214a16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/a0675fd342cc4cdbac02f3dce948ba4c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4c15695765294e2ca6ce0e844df60b29, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/fb4273248ff44ad6a0b57bec50b8bac5, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5f886b1dba1b4bc2bdcea3a39dca8fb3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/da1d0430161644278f6e410c2fd536e1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5de88ab28918402baa6728fa0e98f21f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/512bbdbe6d444926825bab72b64051ff, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9a0c448e6f934ef4a430e0fba6793d53, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/1ca0d5a20ccd483a8c967ccfbee23d9d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/b81a25b53f614f9889c2a6778ef212cd] to archive 2024-11-10T18:16:09,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:16:09,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/7e76feae7e2c4b96be7204ecdf47f673 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/7e76feae7e2c4b96be7204ecdf47f673 2024-11-10T18:16:09,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/def3c05868954003bb7f413908f020c2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/def3c05868954003bb7f413908f020c2 2024-11-10T18:16:09,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e709000c94ac47bba8849f74ea9c615c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e709000c94ac47bba8849f74ea9c615c 2024-11-10T18:16:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e0d9daee4965492a88737a1085fbc13f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e0d9daee4965492a88737a1085fbc13f 2024-11-10T18:16:09,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/cdf73db9b67b45fc91f020ffa755ff2b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/cdf73db9b67b45fc91f020ffa755ff2b 2024-11-10T18:16:09,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8bcea8cb6d7d4734aff835f32bccc066 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8bcea8cb6d7d4734aff835f32bccc066 2024-11-10T18:16:09,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/6e0718ee4bad4d3789c4d0b87e0ff4bf to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/6e0718ee4bad4d3789c4d0b87e0ff4bf 2024-11-10T18:16:09,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9e25497d425e451f97bc00433c65f5e7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9e25497d425e451f97bc00433c65f5e7 2024-11-10T18:16:09,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e4c775a661ec4426a79cf1eee3c7f06c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/e4c775a661ec4426a79cf1eee3c7f06c 2024-11-10T18:16:09,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8c51115b8508444cbb010d10608ee237 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/8c51115b8508444cbb010d10608ee237 2024-11-10T18:16:09,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/84fab6e45c95414ba9fdcf1c9e2b7568 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/84fab6e45c95414ba9fdcf1c9e2b7568 2024-11-10T18:16:09,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/62ff69f9064c48b2a35fc537843dee15 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/62ff69f9064c48b2a35fc537843dee15 2024-11-10T18:16:09,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/f929e1edbdda43b9ac3f5f2967214a16 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/f929e1edbdda43b9ac3f5f2967214a16 2024-11-10T18:16:09,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/a0675fd342cc4cdbac02f3dce948ba4c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/a0675fd342cc4cdbac02f3dce948ba4c 2024-11-10T18:16:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4c15695765294e2ca6ce0e844df60b29 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4c15695765294e2ca6ce0e844df60b29 2024-11-10T18:16:09,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/fb4273248ff44ad6a0b57bec50b8bac5 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/fb4273248ff44ad6a0b57bec50b8bac5 2024-11-10T18:16:09,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5f886b1dba1b4bc2bdcea3a39dca8fb3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5f886b1dba1b4bc2bdcea3a39dca8fb3 2024-11-10T18:16:09,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/da1d0430161644278f6e410c2fd536e1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/da1d0430161644278f6e410c2fd536e1 2024-11-10T18:16:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5de88ab28918402baa6728fa0e98f21f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/5de88ab28918402baa6728fa0e98f21f 2024-11-10T18:16:09,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/512bbdbe6d444926825bab72b64051ff to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/512bbdbe6d444926825bab72b64051ff 2024-11-10T18:16:09,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9a0c448e6f934ef4a430e0fba6793d53 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/9a0c448e6f934ef4a430e0fba6793d53 2024-11-10T18:16:09,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/1ca0d5a20ccd483a8c967ccfbee23d9d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/1ca0d5a20ccd483a8c967ccfbee23d9d 2024-11-10T18:16:09,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/b81a25b53f614f9889c2a6778ef212cd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/b81a25b53f614f9889c2a6778ef212cd 2024-11-10T18:16:09,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cccc080998bb4cdbaaeca0d38063afe1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/f32f1ce94e7f4b81879ac9e55ca17682, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/520cdd3308164f73a88e247ced7b5103, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/389e48079cb344b5bada874a17e898b1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6307276970224e6e9a681db767bfd6ba, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cbbab48f484a4caeb2066b9ca1c34eb4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/d1249eecd80544abaa664ff7d2d51f14, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cb73cb4f6b864884aecf0e6b4106752f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/7e4687c7c5ba424c95d1f904768f8db0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/ee69d028962646d381c6a48a0402d1fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/4aabfe18b0f34eb392a53436e0381831, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/327bfab011c24428a41ca1825d47d8d1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a2f31db52a814395b3de17c21dc9a4a1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/8f3c932b38e94811bea8f00064f58643, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a6629fc035934a2c914284d8eaa5b20f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2ffa2490ca04438c90fe7312dfa70807, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/e95400df86be4004b9f0e26e9c5ffbe9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/c688689f32754185b0995cf16953a785, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3c2cec8c750b4083b63b11db5cb5774d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6849db496bbf44c998dd3fdb26e975fd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/43e5462828ca4ebdb730da3f58154f28, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/06e7cf85d7b245a88b75e996ca718925, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/14d8cb07c52848de9b122c866a024065] to archive 2024-11-10T18:16:09,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:16:09,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cccc080998bb4cdbaaeca0d38063afe1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cccc080998bb4cdbaaeca0d38063afe1 2024-11-10T18:16:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/f32f1ce94e7f4b81879ac9e55ca17682 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/f32f1ce94e7f4b81879ac9e55ca17682 2024-11-10T18:16:09,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/520cdd3308164f73a88e247ced7b5103 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/520cdd3308164f73a88e247ced7b5103 2024-11-10T18:16:09,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/389e48079cb344b5bada874a17e898b1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/389e48079cb344b5bada874a17e898b1 2024-11-10T18:16:09,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6307276970224e6e9a681db767bfd6ba to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6307276970224e6e9a681db767bfd6ba 2024-11-10T18:16:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cbbab48f484a4caeb2066b9ca1c34eb4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cbbab48f484a4caeb2066b9ca1c34eb4 2024-11-10T18:16:09,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/d1249eecd80544abaa664ff7d2d51f14 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/d1249eecd80544abaa664ff7d2d51f14 2024-11-10T18:16:09,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cb73cb4f6b864884aecf0e6b4106752f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/cb73cb4f6b864884aecf0e6b4106752f 2024-11-10T18:16:09,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/7e4687c7c5ba424c95d1f904768f8db0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/7e4687c7c5ba424c95d1f904768f8db0 2024-11-10T18:16:09,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/ee69d028962646d381c6a48a0402d1fb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/ee69d028962646d381c6a48a0402d1fb 2024-11-10T18:16:09,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/4aabfe18b0f34eb392a53436e0381831 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/4aabfe18b0f34eb392a53436e0381831 2024-11-10T18:16:09,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/327bfab011c24428a41ca1825d47d8d1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/327bfab011c24428a41ca1825d47d8d1 2024-11-10T18:16:09,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a2f31db52a814395b3de17c21dc9a4a1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a2f31db52a814395b3de17c21dc9a4a1 2024-11-10T18:16:09,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/8f3c932b38e94811bea8f00064f58643 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/8f3c932b38e94811bea8f00064f58643 2024-11-10T18:16:09,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a6629fc035934a2c914284d8eaa5b20f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/a6629fc035934a2c914284d8eaa5b20f 2024-11-10T18:16:09,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2ffa2490ca04438c90fe7312dfa70807 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2ffa2490ca04438c90fe7312dfa70807 2024-11-10T18:16:09,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/e95400df86be4004b9f0e26e9c5ffbe9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/e95400df86be4004b9f0e26e9c5ffbe9 2024-11-10T18:16:09,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/c688689f32754185b0995cf16953a785 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/c688689f32754185b0995cf16953a785 2024-11-10T18:16:09,601 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3c2cec8c750b4083b63b11db5cb5774d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3c2cec8c750b4083b63b11db5cb5774d 2024-11-10T18:16:09,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6849db496bbf44c998dd3fdb26e975fd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6849db496bbf44c998dd3fdb26e975fd 2024-11-10T18:16:09,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/43e5462828ca4ebdb730da3f58154f28 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/43e5462828ca4ebdb730da3f58154f28 2024-11-10T18:16:09,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/06e7cf85d7b245a88b75e996ca718925 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/06e7cf85d7b245a88b75e996ca718925 2024-11-10T18:16:09,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/14d8cb07c52848de9b122c866a024065 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/14d8cb07c52848de9b122c866a024065 2024-11-10T18:16:09,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/5aad43413be44db69ac1e65a3dac0b35, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a57f7a0f97974f509cf31db4c1ef0b5d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c7dab47031a846c4ae263330215c643f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/dfff82fc47c24d418952a15387866b27, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f469f8526bcd49f9819f5d7016a5372b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2507dda96cf443f1b1f89ff6dfddf01d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0fc0656368174ab5aa4449a1d694f17e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/83d1ddaa9a9f4868a37ccb3423ee23dc, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/954cb0ad30ab4ac68e1ea68f88ee3b42, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d3d1a0f80d6440b7a89742b7fa80390d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d5952e038f68445f90470bb0f835cfbe, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e693ab2015a2474cb6b9b6c842915ebb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/212c87170ce146f19d03e989beedbe1c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c58e99b68eea48a7bb8ef6d216de23b1, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/6085a5b729f04b4c8ecf46ae6b15beb2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7fc85a8908e04433bfa1608bb2a093d0, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2e157cdd5be84192af8a90b208add143, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/53d928da81684aa083990c0d548593b7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f2e92c179db6421cab142c7945a44e32, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/3bfca628a6474d2aba8254a26e169689, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/ff560f81cb6e4ca18af20d2f164d1d1d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e229978dde36478f8806165c0511fb36, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7d2af37d2d974f088ea24294717dfbb8] to archive 2024-11-10T18:16:09,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:16:09,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/5aad43413be44db69ac1e65a3dac0b35 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/5aad43413be44db69ac1e65a3dac0b35 2024-11-10T18:16:09,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a57f7a0f97974f509cf31db4c1ef0b5d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a57f7a0f97974f509cf31db4c1ef0b5d 2024-11-10T18:16:09,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c7dab47031a846c4ae263330215c643f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c7dab47031a846c4ae263330215c643f 2024-11-10T18:16:09,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/dfff82fc47c24d418952a15387866b27 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/dfff82fc47c24d418952a15387866b27 2024-11-10T18:16:09,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f469f8526bcd49f9819f5d7016a5372b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f469f8526bcd49f9819f5d7016a5372b 2024-11-10T18:16:09,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2507dda96cf443f1b1f89ff6dfddf01d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2507dda96cf443f1b1f89ff6dfddf01d 2024-11-10T18:16:09,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0fc0656368174ab5aa4449a1d694f17e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0fc0656368174ab5aa4449a1d694f17e 2024-11-10T18:16:09,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/83d1ddaa9a9f4868a37ccb3423ee23dc to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/83d1ddaa9a9f4868a37ccb3423ee23dc 2024-11-10T18:16:09,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/954cb0ad30ab4ac68e1ea68f88ee3b42 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/954cb0ad30ab4ac68e1ea68f88ee3b42 2024-11-10T18:16:09,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d3d1a0f80d6440b7a89742b7fa80390d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d3d1a0f80d6440b7a89742b7fa80390d 2024-11-10T18:16:09,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d5952e038f68445f90470bb0f835cfbe to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d5952e038f68445f90470bb0f835cfbe 2024-11-10T18:16:09,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e693ab2015a2474cb6b9b6c842915ebb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e693ab2015a2474cb6b9b6c842915ebb 2024-11-10T18:16:09,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/212c87170ce146f19d03e989beedbe1c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/212c87170ce146f19d03e989beedbe1c 2024-11-10T18:16:09,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c58e99b68eea48a7bb8ef6d216de23b1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/c58e99b68eea48a7bb8ef6d216de23b1 2024-11-10T18:16:09,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/6085a5b729f04b4c8ecf46ae6b15beb2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/6085a5b729f04b4c8ecf46ae6b15beb2 2024-11-10T18:16:09,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7fc85a8908e04433bfa1608bb2a093d0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7fc85a8908e04433bfa1608bb2a093d0 2024-11-10T18:16:09,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2e157cdd5be84192af8a90b208add143 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/2e157cdd5be84192af8a90b208add143 2024-11-10T18:16:09,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/53d928da81684aa083990c0d548593b7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/53d928da81684aa083990c0d548593b7 2024-11-10T18:16:09,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f2e92c179db6421cab142c7945a44e32 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/f2e92c179db6421cab142c7945a44e32 2024-11-10T18:16:09,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/3bfca628a6474d2aba8254a26e169689 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/3bfca628a6474d2aba8254a26e169689 2024-11-10T18:16:09,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/ff560f81cb6e4ca18af20d2f164d1d1d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/ff560f81cb6e4ca18af20d2f164d1d1d 2024-11-10T18:16:09,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e229978dde36478f8806165c0511fb36 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/e229978dde36478f8806165c0511fb36 2024-11-10T18:16:09,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7d2af37d2d974f088ea24294717dfbb8 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/7d2af37d2d974f088ea24294717dfbb8 2024-11-10T18:16:09,629 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/recovered.edits/437.seqid, newMaxSeqId=437, maxSeqId=1 2024-11-10T18:16:09,629 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. 2024-11-10T18:16:09,629 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] regionserver.HRegion(1676): Region close journal for 6367e59850ed14d0f0996a4816a8c486: Waiting for close lock at 1731262568327Running coprocessor pre-close hooks at 1731262568327Disabling compacts and flushes for region at 1731262568327Disabling writes for close at 1731262568327Obtaining lock to block concurrent updates at 1731262568327Preparing flush snapshotting stores in 6367e59850ed14d0f0996a4816a8c486 at 1731262568327Finished memstore snapshotting TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731262568327Flushing stores of TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486. at 1731262568328 (+1 ms)Flushing 6367e59850ed14d0f0996a4816a8c486/A: creating writer at 1731262568328Flushing 6367e59850ed14d0f0996a4816a8c486/A: appending metadata at 1731262568330 (+2 ms)Flushing 6367e59850ed14d0f0996a4816a8c486/A: closing flushed file at 1731262568330Flushing 6367e59850ed14d0f0996a4816a8c486/B: creating writer at 1731262568737 (+407 ms)Flushing 6367e59850ed14d0f0996a4816a8c486/B: appending metadata at 1731262568739 (+2 ms)Flushing 6367e59850ed14d0f0996a4816a8c486/B: closing flushed file at 1731262568739Flushing 6367e59850ed14d0f0996a4816a8c486/C: creating writer at 1731262569145 (+406 ms)Flushing 6367e59850ed14d0f0996a4816a8c486/C: appending metadata at 1731262569147 (+2 ms)Flushing 6367e59850ed14d0f0996a4816a8c486/C: closing flushed file at 1731262569147Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4925a52f: reopening flushed file at 1731262569554 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@114a6a6b: reopening flushed file at 1731262569558 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c5e3642: reopening flushed file at 1731262569561 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6367e59850ed14d0f0996a4816a8c486 in 1238ms, sequenceid=434, compaction requested=true at 1731262569565 (+4 ms)Writing region close event to WAL at 1731262569626 (+61 ms)Running coprocessor post-close hooks at 1731262569629 (+3 ms)Closed at 1731262569629 2024-11-10T18:16:09,631 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=132}] handler.UnassignRegionHandler(157): Closed 6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:16:09,631 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=131 updating hbase:meta row=6367e59850ed14d0f0996a4816a8c486, regionState=CLOSED 2024-11-10T18:16:09,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=132, ppid=131, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:16:09,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-10T18:16:09,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseRegionProcedure 6367e59850ed14d0f0996a4816a8c486, server=691ea56905d2,32909,1731262402364 in 1.4580 sec 2024-11-10T18:16:09,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=131, resume processing ppid=130 2024-11-10T18:16:09,636 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, ppid=130, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6367e59850ed14d0f0996a4816a8c486, UNASSIGN in 1.4620 sec 2024-11-10T18:16:09,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-10T18:16:09,637 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4650 sec 2024-11-10T18:16:09,638 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262569638"}]},"ts":"1731262569638"} 2024-11-10T18:16:09,639 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T18:16:09,639 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T18:16:09,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4730 sec 2024-11-10T18:16:10,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-10T18:16:10,315 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:10,315 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T18:16:10,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,317 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=133, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T18:16:10,318 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=133, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,320 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:16:10,321 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/recovered.edits] 2024-11-10T18:16:10,323 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/15ae235fe105403ba22283ac73ed09ac to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/15ae235fe105403ba22283ac73ed09ac 2024-11-10T18:16:10,324 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4493d4becef64ea6baab7f52cc420408 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/4493d4becef64ea6baab7f52cc420408 2024-11-10T18:16:10,325 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/951d46dee26447ca81b32290da3ac53b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/951d46dee26447ca81b32290da3ac53b 2024-11-10T18:16:10,326 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/d1d6945ef8ca49f6977d5144852f50cb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/A/d1d6945ef8ca49f6977d5144852f50cb 2024-11-10T18:16:10,327 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/26a4449e8dc7410a94be7e126f0c56f0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/26a4449e8dc7410a94be7e126f0c56f0 2024-11-10T18:16:10,328 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2954cada57b44242a08e2b9ea44246d0 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/2954cada57b44242a08e2b9ea44246d0 2024-11-10T18:16:10,329 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3d1338c039c24d379c0ea99189db6712 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/3d1338c039c24d379c0ea99189db6712 2024-11-10T18:16:10,330 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6f971f2bb608458590ed2971a059c866 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/B/6f971f2bb608458590ed2971a059c866 2024-11-10T18:16:10,331 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0deee92da284424d9ac8111794d1cb66 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/0deee92da284424d9ac8111794d1cb66 2024-11-10T18:16:10,332 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/54064386f65c417681be5e5dfe0d9674 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/54064386f65c417681be5e5dfe0d9674 2024-11-10T18:16:10,333 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a845467c7c1d4358b4bfba69c6ead282 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/a845467c7c1d4358b4bfba69c6ead282 2024-11-10T18:16:10,334 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d8f56ef86fc640d9bb1653942bc0473e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/C/d8f56ef86fc640d9bb1653942bc0473e 2024-11-10T18:16:10,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/recovered.edits/437.seqid to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486/recovered.edits/437.seqid 2024-11-10T18:16:10,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/6367e59850ed14d0f0996a4816a8c486 2024-11-10T18:16:10,336 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T18:16:10,338 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=133, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,340 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T18:16:10,342 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T18:16:10,342 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=133, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,343 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T18:16:10,343 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731262570343"}]},"ts":"9223372036854775807"} 2024-11-10T18:16:10,344 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T18:16:10,344 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6367e59850ed14d0f0996a4816a8c486, NAME => 'TestAcidGuarantees,,1731262536495.6367e59850ed14d0f0996a4816a8c486.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T18:16:10,344 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T18:16:10,345 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731262570344"}]},"ts":"9223372036854775807"} 2024-11-10T18:16:10,346 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T18:16:10,346 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=133, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-11-10T18:16:10,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-10T18:16:10,425 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T18:16:10,425 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:10,434 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247 (was 249), OpenFileDescriptor=441 (was 445), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 258), ProcessCount=11 (was 11), AvailableMemoryMB=7996 (was 8019) 2024-11-10T18:16:10,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=7996 2024-11-10T18:16:10,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:16:10,443 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:16:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:10,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T18:16:10,444 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:10,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 134 2024-11-10T18:16:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-10T18:16:10,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T18:16:10,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742331_1507 (size=1061) 2024-11-10T18:16:10,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-10T18:16:10,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-10T18:16:10,851 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193 2024-11-10T18:16:10,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742332_1508 (size=53) 2024-11-10T18:16:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-10T18:16:11,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:16:11,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 01f94435f1c5387043f08668493d3e9e, disabling compactions & flushes 2024-11-10T18:16:11,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. after waiting 0 ms 2024-11-10T18:16:11,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,256 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,256 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 01f94435f1c5387043f08668493d3e9e: Waiting for close lock at 1731262571256Disabling compacts and flushes for region at 1731262571256Disabling writes for close at 1731262571256Writing region close event to WAL at 1731262571256Closed at 1731262571256 2024-11-10T18:16:11,257 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T18:16:11,258 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731262571257"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731262571257"}]},"ts":"1731262571257"} 2024-11-10T18:16:11,259 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T18:16:11,260 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T18:16:11,260 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262571260"}]},"ts":"1731262571260"} 2024-11-10T18:16:11,261 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-10T18:16:11,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, ASSIGN}] 2024-11-10T18:16:11,262 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, ASSIGN 2024-11-10T18:16:11,262 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, ASSIGN; state=OFFLINE, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=false 2024-11-10T18:16:11,413 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:11,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=134, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, ASSIGN because future has completed 2024-11-10T18:16:11,414 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:16:11,567 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,567 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7752): Opening region: {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:16:11,568 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,568 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:16:11,568 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7794): checking encryption for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,568 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(7797): checking classloading for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,569 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,570 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:16:11,570 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01f94435f1c5387043f08668493d3e9e columnFamilyName A 2024-11-10T18:16:11,570 DEBUG [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:11,570 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(327): Store=01f94435f1c5387043f08668493d3e9e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:16:11,570 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,571 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:16:11,571 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01f94435f1c5387043f08668493d3e9e columnFamilyName B 2024-11-10T18:16:11,571 DEBUG [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:11,572 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(327): Store=01f94435f1c5387043f08668493d3e9e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:16:11,572 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,572 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:16:11,573 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01f94435f1c5387043f08668493d3e9e columnFamilyName C 2024-11-10T18:16:11,573 DEBUG [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:11,573 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(327): Store=01f94435f1c5387043f08668493d3e9e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:16:11,573 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,573 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1038): replaying wal for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,574 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,574 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,574 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1048): stopping wal replay for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,575 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1060): Cleaning up temporary data for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,575 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:16:11,576 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1093): writing seq id for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,577 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T18:16:11,577 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1114): Opened 01f94435f1c5387043f08668493d3e9e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65793792, jitterRate=-0.019596099853515625}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:16:11,577 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:11,578 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegion(1006): Region open journal for 01f94435f1c5387043f08668493d3e9e: Running coprocessor pre-open hook at 1731262571568Writing region info on filesystem at 1731262571568Initializing all the Stores at 1731262571568Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262571568Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262571569 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262571569Cleaning up temporary data from old regions at 1731262571575 (+6 ms)Running coprocessor post-open hooks at 1731262571577 (+2 ms)Region opened successfully at 1731262571578 (+1 ms) 2024-11-10T18:16:11,578 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., pid=136, masterSystemTime=1731262571565 2024-11-10T18:16:11,580 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,580 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=136}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:11,580 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=135 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=OPEN, openSeqNum=2, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:11,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=136, ppid=135, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:16:11,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-10T18:16:11,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; OpenRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 in 168 msec 2024-11-10T18:16:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-10T18:16:11,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=134 2024-11-10T18:16:11,585 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=134, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, ASSIGN in 323 msec 2024-11-10T18:16:11,586 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T18:16:11,586 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262571586"}]},"ts":"1731262571586"} 2024-11-10T18:16:11,587 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-10T18:16:11,587 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=134, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T18:16:11,588 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1440 sec 2024-11-10T18:16:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=134 2024-11-10T18:16:12,595 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:12,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@185b2823, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:12,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:12,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:12,597 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:12,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251011cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:12,598 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:12,599 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bc5ef23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:12,600 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:12,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60382, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:12,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T18:16:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 691ea56905d2,41343,1731262401541 2024-11-10T18:16:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7f5ae525 2024-11-10T18:16:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T18:16:12,603 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T18:16:12,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-10T18:16:12,604 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T18:16:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:12,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742333_1509 (size=1097) 2024-11-10T18:16:13,014 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-10T18:16:13,015 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-10T18:16:13,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:16:13,018 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, REOPEN/MOVE}] 2024-11-10T18:16:13,019 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, REOPEN/MOVE 2024-11-10T18:16:13,020 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,021 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, REOPEN/MOVE because future has completed 2024-11-10T18:16:13,021 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:16:13,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:16:13,173 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,173 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:16:13,173 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 01f94435f1c5387043f08668493d3e9e, disabling compactions & flushes 2024-11-10T18:16:13,173 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,173 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,173 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. after waiting 0 ms 2024-11-10T18:16:13,173 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,176 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-10T18:16:13,177 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,177 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 01f94435f1c5387043f08668493d3e9e: Waiting for close lock at 1731262573173Running coprocessor pre-close hooks at 1731262573173Disabling compacts and flushes for region at 1731262573173Disabling writes for close at 1731262573173Writing region close event to WAL at 1731262573174 (+1 ms)Running coprocessor post-close hooks at 1731262573177 (+3 ms)Closed at 1731262573177 2024-11-10T18:16:13,177 WARN [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionServer(3299): Not adding moved region record: 01f94435f1c5387043f08668493d3e9e to self. 2024-11-10T18:16:13,178 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,179 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=CLOSED 2024-11-10T18:16:13,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:16:13,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-10T18:16:13,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 in 159 msec 2024-11-10T18:16:13,182 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, REOPEN/MOVE; state=CLOSED, location=691ea56905d2,32909,1731262402364; forceNewPlan=false, retain=true 2024-11-10T18:16:13,333 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=OPENING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,334 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, REOPEN/MOVE because future has completed 2024-11-10T18:16:13,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:16:13,488 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,488 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7752): Opening region: {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} 2024-11-10T18:16:13,489 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,489 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T18:16:13,489 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7794): checking encryption for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,489 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(7797): checking classloading for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,490 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,490 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:16:13,491 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01f94435f1c5387043f08668493d3e9e columnFamilyName A 2024-11-10T18:16:13,492 DEBUG [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:13,492 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(327): Store=01f94435f1c5387043f08668493d3e9e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:16:13,492 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,493 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:16:13,493 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01f94435f1c5387043f08668493d3e9e columnFamilyName B 2024-11-10T18:16:13,493 DEBUG [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:13,493 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(327): Store=01f94435f1c5387043f08668493d3e9e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:16:13,493 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,494 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-10T18:16:13,494 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 01f94435f1c5387043f08668493d3e9e columnFamilyName C 2024-11-10T18:16:13,494 DEBUG [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:13,494 INFO [StoreOpener-01f94435f1c5387043f08668493d3e9e-1 {}] regionserver.HStore(327): Store=01f94435f1c5387043f08668493d3e9e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T18:16:13,494 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,494 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1038): replaying wal for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,495 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,496 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,497 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1048): stopping wal replay for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,497 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1060): Cleaning up temporary data for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,497 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-10T18:16:13,498 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1093): writing seq id for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,499 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1114): Opened 01f94435f1c5387043f08668493d3e9e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67758178, jitterRate=0.009675532579421997}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-10T18:16:13,499 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,500 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegion(1006): Region open journal for 01f94435f1c5387043f08668493d3e9e: Running coprocessor pre-open hook at 1731262573489Writing region info on filesystem at 1731262573489Initializing all the Stores at 1731262573489Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262573490 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262573490Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731262573490Cleaning up temporary data from old regions at 1731262573497 (+7 ms)Running coprocessor post-open hooks at 1731262573499 (+2 ms)Region opened successfully at 1731262573500 (+1 ms) 2024-11-10T18:16:13,500 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., pid=141, masterSystemTime=1731262573486 2024-11-10T18:16:13,502 DEBUG [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,502 INFO [RS_OPEN_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_OPEN_REGION, pid=141}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,502 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=OPEN, openSeqNum=5, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,504 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=139, state=RUNNABLE, hasLock=false; OpenRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:16:13,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=139 2024-11-10T18:16:13,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=139, state=SUCCESS, hasLock=false; OpenRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 in 170 msec 2024-11-10T18:16:13,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-10T18:16:13,507 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, REOPEN/MOVE in 488 msec 2024-11-10T18:16:13,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-10T18:16:13,509 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 491 msec 2024-11-10T18:16:13,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 904 msec 2024-11-10T18:16:13,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-10T18:16:13,511 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:13,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,513 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d83ba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,514 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50676, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7de7c6e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,515 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,515 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,515 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,516 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10b1dbed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,517 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41e91df5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,518 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,518 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,518 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,518 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,518 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f29d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,519 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1870c8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aae898a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,522 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe86b3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,523 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,523 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,523 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,523 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f8ca1d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,523 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,524 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1198891e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,525 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,525 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,526 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34a8b0aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,527 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,527 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16977c3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,528 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,528 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,528 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6e1417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,529 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2938718d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,530 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,530 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,531 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5423759a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,532 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4b5a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,533 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,533 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,533 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e85e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,534 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33606abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e766e1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 691ea56905d2,41343,-1 for getting cluster id 2024-11-10T18:16:13,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T18:16:13,536 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a8ca52b0-39cc-4209-9f20-fdad45deec70' 2024-11-10T18:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T18:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a8ca52b0-39cc-4209-9f20-fdad45deec70" 2024-11-10T18:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2c5bb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [691ea56905d2,41343,-1] 2024-11-10T18:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T18:16:13,536 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:13,537 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T18:16:13,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c80493, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T18:16:13,538 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,538 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,538 DEBUG [Thread-2299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,538 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,538 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,539 DEBUG [Thread-2312 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,539 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,539 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60402, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60428, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-10T18:16:13,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60438, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T18:16:13,542 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,542 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:13,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:13,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,545 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,549 DEBUG [Thread-2301 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,549 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:16:13,549 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,557 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,557 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T18:16:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,558 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=691ea56905d2,32909,1731262402364, seqNum=-1] 2024-11-10T18:16:13,561 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T18:16:13,562 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,563 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60478, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T18:16:13,565 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5] 2024-11-10T18:16:13,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731262573563, max timestamp=1731262573574Num uniques 859; 2024-11-10T18:16:13,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-10T18:16:13,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262573563, max timestamp=1731262573574Num uniques 859; 2024-11-10T18:16:13,579 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:16:13,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731262573563, max timestamp=1731262573574Num uniques 859; 2024-11-10T18:16:13,582 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-10T18:16:13,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111086ba5f40e2d6428b945aa22ef34b4b81_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262573548/Put/seqid=0 2024-11-10T18:16:13,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742334_1510 (size=14594) 2024-11-10T18:16:13,599 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:13,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111086ba5f40e2d6428b945aa22ef34b4b81_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111086ba5f40e2d6428b945aa22ef34b4b81_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:13,604 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7bfe5dc3c7db42e3a2bb4da609538d57, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:13,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7bfe5dc3c7db42e3a2bb4da609538d57 is 175, key is test_row_0/A:col10/1731262573548/Put/seqid=0 2024-11-10T18:16:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262633598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262633603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262633604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262633605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262633605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742335_1511 (size=39545) 2024-11-10T18:16:13,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T18:16:13,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:13,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:13,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:13,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262633715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262633715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262633715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262633715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262633715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,720 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:13,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:13,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T18:16:13,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262633924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262633924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262633925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262633925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,928 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:13,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262633925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:13,929 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:14,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:14,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,021 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7bfe5dc3c7db42e3a2bb4da609538d57 2024-11-10T18:16:14,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/42a3972e9431402386f85237aebc6aa9 is 50, key is test_row_0/B:col10/1731262573548/Put/seqid=0 2024-11-10T18:16:14,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742336_1512 (size=9657) 2024-11-10T18:16:14,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:14,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:14,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T18:16:14,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262634234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262634234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262634235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262634235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262634235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,240 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:14,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:14,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/42a3972e9431402386f85237aebc6aa9 2024-11-10T18:16:14,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:14,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:14,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] handler.RSProcedureHandler(58): pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=143 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=143 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:14,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/b53f0b1f83554825bdc01f61a0584deb is 50, key is test_row_0/C:col10/1731262573548/Put/seqid=0 2024-11-10T18:16:14,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742337_1513 (size=9657) 2024-11-10T18:16:14,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/b53f0b1f83554825bdc01f61a0584deb 2024-11-10T18:16:14,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7bfe5dc3c7db42e3a2bb4da609538d57 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57 2024-11-10T18:16:14,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57, entries=200, sequenceid=14, filesize=38.6 K 2024-11-10T18:16:14,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/42a3972e9431402386f85237aebc6aa9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/42a3972e9431402386f85237aebc6aa9 2024-11-10T18:16:14,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/42a3972e9431402386f85237aebc6aa9, entries=100, sequenceid=14, filesize=9.4 K 2024-11-10T18:16:14,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/b53f0b1f83554825bdc01f61a0584deb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/b53f0b1f83554825bdc01f61a0584deb 2024-11-10T18:16:14,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/b53f0b1f83554825bdc01f61a0584deb, entries=100, sequenceid=14, filesize=9.4 K 2024-11-10T18:16:14,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 01f94435f1c5387043f08668493d3e9e in 937ms, sequenceid=14, compaction requested=false 2024-11-10T18:16:14,487 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-10T18:16:14,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:14,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-10T18:16:14,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:14,614 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-10T18:16:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:14,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:14,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e1ba973997c4d62b466b7d46f70f418_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262573604/Put/seqid=0 2024-11-10T18:16:14,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742338_1514 (size=12154) 2024-11-10T18:16:14,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T18:16:14,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:14,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:14,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262634748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262634749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262634750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,755 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,755 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262634750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,756 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262634753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262634864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262634865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262634865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:14,868 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:14,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:15,038 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102e1ba973997c4d62b466b7d46f70f418_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e1ba973997c4d62b466b7d46f70f418_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:15,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/af3449ab395646d7a0adffa4d8a02b04, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:15,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/af3449ab395646d7a0adffa4d8a02b04 is 175, key is test_row_0/A:col10/1731262573604/Put/seqid=0 2024-11-10T18:16:15,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742339_1515 (size=30955) 2024-11-10T18:16:15,043 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/af3449ab395646d7a0adffa4d8a02b04 2024-11-10T18:16:15,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/169b829c9c024a38a50248d3048ef40b is 50, key is test_row_0/B:col10/1731262573604/Put/seqid=0 2024-11-10T18:16:15,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742340_1516 (size=12001) 2024-11-10T18:16:15,053 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/169b829c9c024a38a50248d3048ef40b 2024-11-10T18:16:15,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/97885dc128984010912da1cc327fb2b6 is 50, key is test_row_0/C:col10/1731262573604/Put/seqid=0 2024-11-10T18:16:15,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742341_1517 (size=12001) 2024-11-10T18:16:15,062 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/97885dc128984010912da1cc327fb2b6 2024-11-10T18:16:15,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/af3449ab395646d7a0adffa4d8a02b04 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04 2024-11-10T18:16:15,077 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04, entries=150, sequenceid=51, filesize=30.2 K 2024-11-10T18:16:15,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/169b829c9c024a38a50248d3048ef40b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/169b829c9c024a38a50248d3048ef40b 2024-11-10T18:16:15,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262635074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262635075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262635075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262635075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262635075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,082 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/169b829c9c024a38a50248d3048ef40b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T18:16:15,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/97885dc128984010912da1cc327fb2b6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/97885dc128984010912da1cc327fb2b6 2024-11-10T18:16:15,086 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/97885dc128984010912da1cc327fb2b6, entries=150, sequenceid=51, filesize=11.7 K 2024-11-10T18:16:15,087 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 01f94435f1c5387043f08668493d3e9e in 474ms, sequenceid=51, compaction requested=false 2024-11-10T18:16:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:15,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-10T18:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-10T18:16:15,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-10T18:16:15,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5450 sec 2024-11-10T18:16:15,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.5520 sec 2024-11-10T18:16:15,149 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T18:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:15,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:16:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:15,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:15,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2cb6b3d1d3049f08da5ca4649afc615_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:15,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742342_1518 (size=12154) 2024-11-10T18:16:15,399 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:15,402 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2cb6b3d1d3049f08da5ca4649afc615_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2cb6b3d1d3049f08da5ca4649afc615_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:15,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/51bee30119044fed933ac10f50802746, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:15,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/51bee30119044fed933ac10f50802746 is 175, key is test_row_0/A:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:15,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742343_1519 (size=30955) 2024-11-10T18:16:15,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262635408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,414 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262575395, max timestamp=1731262575413Num uniques 859; 2024-11-10T18:16:15,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:16:15,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262575395, max timestamp=1731262575413Num uniques 859; 2024-11-10T18:16:15,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:16:15,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731262575395, max timestamp=1731262575413Num uniques 859; 2024-11-10T18:16:15,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:16:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262635434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262635435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262635436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262635437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262635524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262635544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262635544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262635555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262635555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-10T18:16:15,694 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:15,695 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-10T18:16:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:15,697 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:15,697 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:15,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:15,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262635734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262635755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262635755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262635765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262635765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:15,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:15,808 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/51bee30119044fed933ac10f50802746 2024-11-10T18:16:15,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/17119f4305814bfaab00dda27081bade is 50, key is test_row_0/B:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:15,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742344_1520 (size=12001) 2024-11-10T18:16:15,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:15,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:15,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:15,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:15,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:15,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:15,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:16,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:16,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:16,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262636044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,050 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262636065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262636065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,070 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262636075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262636075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:16,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:16,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/17119f4305814bfaab00dda27081bade 2024-11-10T18:16:16,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/6839dfc42d454e72962fa9b4372717e3 is 50, key is test_row_0/C:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:16,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742345_1521 (size=12001) 2024-11-10T18:16:16,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:16,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:16,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:16,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262636555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262636575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262636575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,578 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262636584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262636584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:16,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:16,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:16,628 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/6839dfc42d454e72962fa9b4372717e3 2024-11-10T18:16:16,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/51bee30119044fed933ac10f50802746 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746 2024-11-10T18:16:16,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746, entries=150, sequenceid=63, filesize=30.2 K 2024-11-10T18:16:16,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/17119f4305814bfaab00dda27081bade as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/17119f4305814bfaab00dda27081bade 2024-11-10T18:16:16,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/17119f4305814bfaab00dda27081bade, entries=150, sequenceid=63, filesize=11.7 K 2024-11-10T18:16:16,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/6839dfc42d454e72962fa9b4372717e3 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/6839dfc42d454e72962fa9b4372717e3 2024-11-10T18:16:16,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/6839dfc42d454e72962fa9b4372717e3, entries=150, sequenceid=63, filesize=11.7 K 2024-11-10T18:16:16,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 01f94435f1c5387043f08668493d3e9e in 1254ms, sequenceid=63, compaction requested=true 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:16,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:16,642 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:16,642 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:16,643 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:16,643 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:16,643 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:16,643 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:16,643 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,643 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,644 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=99.1 K 2024-11-10T18:16:16,644 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/42a3972e9431402386f85237aebc6aa9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/169b829c9c024a38a50248d3048ef40b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/17119f4305814bfaab00dda27081bade] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=32.9 K 2024-11-10T18:16:16,644 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,644 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746] 2024-11-10T18:16:16,644 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bfe5dc3c7db42e3a2bb4da609538d57, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731262573546 2024-11-10T18:16:16,644 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 42a3972e9431402386f85237aebc6aa9, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731262573546 2024-11-10T18:16:16,644 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting af3449ab395646d7a0adffa4d8a02b04, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731262573598 2024-11-10T18:16:16,644 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 169b829c9c024a38a50248d3048ef40b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731262573598 2024-11-10T18:16:16,645 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51bee30119044fed933ac10f50802746, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731262575387 2024-11-10T18:16:16,645 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 17119f4305814bfaab00dda27081bade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731262575387 2024-11-10T18:16:16,650 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:16,651 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:16,652 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/7a811001447a4e608992cbd4c53beb16 is 50, key is test_row_0/B:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:16,657 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411100a925b0b68c54a95a50fdcaa51c4c5ea_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:16,658 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411100a925b0b68c54a95a50fdcaa51c4c5ea_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:16,659 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100a925b0b68c54a95a50fdcaa51c4c5ea_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:16,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742346_1522 (size=12104) 2024-11-10T18:16:16,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742347_1523 (size=4469) 2024-11-10T18:16:16,672 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/7a811001447a4e608992cbd4c53beb16 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7a811001447a4e608992cbd4c53beb16 2024-11-10T18:16:16,676 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#445 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:16,676 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/bb1e1d546af241078520ea69d3a049fb is 175, key is test_row_0/A:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:16,679 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into 7a811001447a4e608992cbd4c53beb16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:16,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:16,679 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=13, startTime=1731262576642; duration=0sec 2024-11-10T18:16:16,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:16,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:16,679 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:16,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:16,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:16,680 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,681 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/b53f0b1f83554825bdc01f61a0584deb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/97885dc128984010912da1cc327fb2b6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/6839dfc42d454e72962fa9b4372717e3] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=32.9 K 2024-11-10T18:16:16,681 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting b53f0b1f83554825bdc01f61a0584deb, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731262573546 2024-11-10T18:16:16,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742348_1524 (size=31058) 2024-11-10T18:16:16,682 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 97885dc128984010912da1cc327fb2b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731262573598 2024-11-10T18:16:16,683 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 6839dfc42d454e72962fa9b4372717e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731262575387 2024-11-10T18:16:16,690 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:16,690 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/bae1c7ad11ba4036a8be09b68b21945a is 50, key is test_row_0/C:col10/1731262575387/Put/seqid=0 2024-11-10T18:16:16,694 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/bb1e1d546af241078520ea69d3a049fb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/bb1e1d546af241078520ea69d3a049fb 2024-11-10T18:16:16,698 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into bb1e1d546af241078520ea69d3a049fb(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:16,698 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:16,698 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=13, startTime=1731262576642; duration=0sec 2024-11-10T18:16:16,698 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:16,698 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:16,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742349_1525 (size=12104) 2024-11-10T18:16:16,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-10T18:16:16,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:16,765 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-10T18:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:16,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:16,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d9c312b05a314b65bc780e21a8b72d0a_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262575434/Put/seqid=0 2024-11-10T18:16:16,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742350_1526 (size=12154) 2024-11-10T18:16:16,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:17,106 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/bae1c7ad11ba4036a8be09b68b21945a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/bae1c7ad11ba4036a8be09b68b21945a 2024-11-10T18:16:17,114 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into bae1c7ad11ba4036a8be09b68b21945a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:17,114 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:17,114 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=13, startTime=1731262576642; duration=0sec 2024-11-10T18:16:17,114 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:17,114 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:17,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:17,181 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d9c312b05a314b65bc780e21a8b72d0a_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9c312b05a314b65bc780e21a8b72d0a_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:17,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/e97c24dcb49342eb8a051f2f9b3e4d7c, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:17,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/e97c24dcb49342eb8a051f2f9b3e4d7c is 175, key is test_row_0/A:col10/1731262575434/Put/seqid=0 2024-11-10T18:16:17,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742351_1527 (size=30955) 2024-11-10T18:16:17,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:17,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:17,589 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/e97c24dcb49342eb8a051f2f9b3e4d7c 2024-11-10T18:16:17,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/4755806e9ef64f60b31c896702313820 is 50, key is test_row_0/B:col10/1731262575434/Put/seqid=0 2024-11-10T18:16:17,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742352_1528 (size=12001) 2024-11-10T18:16:17,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262637595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,601 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/4755806e9ef64f60b31c896702313820 2024-11-10T18:16:17,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262637595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262637595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,601 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,601 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,601 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262637599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262637602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/f75a0cc4d0a94fd199a4986b9aeedd48 is 50, key is test_row_0/C:col10/1731262575434/Put/seqid=0 2024-11-10T18:16:17,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742353_1529 (size=12001) 2024-11-10T18:16:17,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262637714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262637714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,718 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262637925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:17,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262637925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,928 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:17,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,015 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/f75a0cc4d0a94fd199a4986b9aeedd48 2024-11-10T18:16:18,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/e97c24dcb49342eb8a051f2f9b3e4d7c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c 2024-11-10T18:16:18,022 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c, entries=150, sequenceid=100, filesize=30.2 K 2024-11-10T18:16:18,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/4755806e9ef64f60b31c896702313820 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/4755806e9ef64f60b31c896702313820 2024-11-10T18:16:18,026 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/4755806e9ef64f60b31c896702313820, entries=150, sequenceid=100, filesize=11.7 K 2024-11-10T18:16:18,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/f75a0cc4d0a94fd199a4986b9aeedd48 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/f75a0cc4d0a94fd199a4986b9aeedd48 2024-11-10T18:16:18,030 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/f75a0cc4d0a94fd199a4986b9aeedd48, entries=150, sequenceid=100, filesize=11.7 K 2024-11-10T18:16:18,031 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 01f94435f1c5387043f08668493d3e9e in 1265ms, sequenceid=100, compaction requested=false 2024-11-10T18:16:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:18,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-10T18:16:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-10T18:16:18,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-10T18:16:18,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3350 sec 2024-11-10T18:16:18,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.3390 sec 2024-11-10T18:16:18,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:18,238 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:18,238 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:18,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111047623871f0d74e7690e8c7be2178d325_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:18,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742354_1530 (size=14594) 2024-11-10T18:16:18,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262638283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262638284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262638395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262638395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262638605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262638605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,649 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:18,652 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111047623871f0d74e7690e8c7be2178d325_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111047623871f0d74e7690e8c7be2178d325_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:18,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33631d85a1464f3a8340d9c5674977ea, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:18,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33631d85a1464f3a8340d9c5674977ea is 175, key is test_row_0/A:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:18,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742355_1531 (size=39549) 2024-11-10T18:16:18,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262638915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,918 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:18,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262638915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=114, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33631d85a1464f3a8340d9c5674977ea 2024-11-10T18:16:19,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/65fa20f379b446fdb16ef61fa77a7fa7 is 50, key is test_row_0/B:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:19,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742356_1532 (size=12001) 2024-11-10T18:16:19,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262639425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262639434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/65fa20f379b446fdb16ef61fa77a7fa7 2024-11-10T18:16:19,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/340563790411484e84bcbe790d296c18 is 50, key is test_row_0/C:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:19,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742357_1533 (size=12001) 2024-11-10T18:16:19,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/340563790411484e84bcbe790d296c18 2024-11-10T18:16:19,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33631d85a1464f3a8340d9c5674977ea as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea 2024-11-10T18:16:19,497 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea, entries=200, sequenceid=114, filesize=38.6 K 2024-11-10T18:16:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/65fa20f379b446fdb16ef61fa77a7fa7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/65fa20f379b446fdb16ef61fa77a7fa7 2024-11-10T18:16:19,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/65fa20f379b446fdb16ef61fa77a7fa7, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T18:16:19,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/340563790411484e84bcbe790d296c18 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/340563790411484e84bcbe790d296c18 2024-11-10T18:16:19,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/340563790411484e84bcbe790d296c18, entries=150, sequenceid=114, filesize=11.7 K 2024-11-10T18:16:19,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 01f94435f1c5387043f08668493d3e9e in 1268ms, sequenceid=114, compaction requested=true 2024-11-10T18:16:19,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:19,506 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:19,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:19,507 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:19,507 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:19,507 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:19,508 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:19,508 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/bb1e1d546af241078520ea69d3a049fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=99.2 K 2024-11-10T18:16:19,508 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:19,508 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/bb1e1d546af241078520ea69d3a049fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea] 2024-11-10T18:16:19,508 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:19,508 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb1e1d546af241078520ea69d3a049fb, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731262575387 2024-11-10T18:16:19,508 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:19,508 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:19,509 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7a811001447a4e608992cbd4c53beb16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/4755806e9ef64f60b31c896702313820, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/65fa20f379b446fdb16ef61fa77a7fa7] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=35.3 K 2024-11-10T18:16:19,509 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting e97c24dcb49342eb8a051f2f9b3e4d7c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731262575434 2024-11-10T18:16:19,509 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33631d85a1464f3a8340d9c5674977ea, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731262577593 2024-11-10T18:16:19,509 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a811001447a4e608992cbd4c53beb16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731262575387 2024-11-10T18:16:19,510 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 4755806e9ef64f60b31c896702313820, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731262575434 2024-11-10T18:16:19,511 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fa20f379b446fdb16ef61fa77a7fa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731262577598 2024-11-10T18:16:19,515 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:19,519 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110309263d4a44d45d8af4151226d96c5d5_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:19,520 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:19,521 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/16ab32ae684148ff8b8f4133e44b06fb is 50, key is test_row_0/B:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:19,521 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110309263d4a44d45d8af4151226d96c5d5_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:19,522 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110309263d4a44d45d8af4151226d96c5d5_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:19,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742358_1534 (size=12207) 2024-11-10T18:16:19,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742359_1535 (size=4469) 2024-11-10T18:16:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:19,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:16:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:19,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:19,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2b1e34c65dd4ae19b37fa00cb6c6c50_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262578282/Put/seqid=0 2024-11-10T18:16:19,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742360_1536 (size=14744) 2024-11-10T18:16:19,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262639625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262639626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,632 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262639632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,638 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262639735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262639745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262639745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,748 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-10T18:16:19,855 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:19,856 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-10T18:16:19,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T18:16:19,857 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:19,858 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:19,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:19,926 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#453 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:19,927 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/f234d3f2d5854b78bba984854cb60b33 is 175, key is test_row_0/A:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:19,929 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/16ab32ae684148ff8b8f4133e44b06fb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/16ab32ae684148ff8b8f4133e44b06fb 2024-11-10T18:16:19,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742361_1537 (size=31161) 2024-11-10T18:16:19,936 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into 16ab32ae684148ff8b8f4133e44b06fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:19,936 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:19,936 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=13, startTime=1731262579507; duration=0sec 2024-11-10T18:16:19,936 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:19,937 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:19,937 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:19,937 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:19,937 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:19,937 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:19,937 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/bae1c7ad11ba4036a8be09b68b21945a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/f75a0cc4d0a94fd199a4986b9aeedd48, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/340563790411484e84bcbe790d296c18] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=35.3 K 2024-11-10T18:16:19,938 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting bae1c7ad11ba4036a8be09b68b21945a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731262575387 2024-11-10T18:16:19,938 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting f75a0cc4d0a94fd199a4986b9aeedd48, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731262575434 2024-11-10T18:16:19,938 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 340563790411484e84bcbe790d296c18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731262577598 2024-11-10T18:16:19,943 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:19,943 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/dfa21e1cf9b04a56b2dcf403407ed39f is 50, key is test_row_0/C:col10/1731262577598/Put/seqid=0 2024-11-10T18:16:19,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262639944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742362_1538 (size=12207) 2024-11-10T18:16:19,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262639954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262639954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:19,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T18:16:20,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,024 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:20,027 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d2b1e34c65dd4ae19b37fa00cb6c6c50_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2b1e34c65dd4ae19b37fa00cb6c6c50_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:20,028 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/aeb7e3aad68d493387ed8b4e5f09c1b9, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:20,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/aeb7e3aad68d493387ed8b4e5f09c1b9 is 175, key is test_row_0/A:col10/1731262578282/Put/seqid=0 2024-11-10T18:16:20,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742363_1539 (size=39699) 2024-11-10T18:16:20,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:20,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T18:16:20,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262640255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262640264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262640264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,337 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/f234d3f2d5854b78bba984854cb60b33 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/f234d3f2d5854b78bba984854cb60b33 2024-11-10T18:16:20,340 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into f234d3f2d5854b78bba984854cb60b33(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:20,340 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:20,340 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=13, startTime=1731262579506; duration=0sec 2024-11-10T18:16:20,341 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:20,341 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:20,351 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/dfa21e1cf9b04a56b2dcf403407ed39f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/dfa21e1cf9b04a56b2dcf403407ed39f 2024-11-10T18:16:20,354 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into dfa21e1cf9b04a56b2dcf403407ed39f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:20,354 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:20,354 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=13, startTime=1731262579507; duration=0sec 2024-11-10T18:16:20,355 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:20,355 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:20,433 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/aeb7e3aad68d493387ed8b4e5f09c1b9 2024-11-10T18:16:20,439 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/59244f0ac39143edbb7451b6677ae739 is 50, key is test_row_0/B:col10/1731262578282/Put/seqid=0 2024-11-10T18:16:20,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742364_1540 (size=12151) 2024-11-10T18:16:20,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262640445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,449 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262640455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,459 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T18:16:20,493 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-10T18:16:20,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262640764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262640774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262640775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:20,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/59244f0ac39143edbb7451b6677ae739 2024-11-10T18:16:20,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/5b9612c7508349dda360ee1ecee1a034 is 50, key is test_row_0/C:col10/1731262578282/Put/seqid=0 2024-11-10T18:16:20,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742365_1541 (size=12151) 2024-11-10T18:16:20,854 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/5b9612c7508349dda360ee1ecee1a034 2024-11-10T18:16:20,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/aeb7e3aad68d493387ed8b4e5f09c1b9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9 2024-11-10T18:16:20,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9, entries=200, sequenceid=134, filesize=38.8 K 2024-11-10T18:16:20,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/59244f0ac39143edbb7451b6677ae739 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/59244f0ac39143edbb7451b6677ae739 2024-11-10T18:16:20,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/59244f0ac39143edbb7451b6677ae739, entries=150, sequenceid=134, filesize=11.9 K 2024-11-10T18:16:20,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/5b9612c7508349dda360ee1ecee1a034 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5b9612c7508349dda360ee1ecee1a034 2024-11-10T18:16:20,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5b9612c7508349dda360ee1ecee1a034, entries=150, sequenceid=134, filesize=11.9 K 2024-11-10T18:16:20,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 01f94435f1c5387043f08668493d3e9e in 1261ms, sequenceid=134, compaction requested=false 2024-11-10T18:16:20,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:20,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:20,929 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:20,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:20,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dcb6bc304d794a16821386420e9022b8_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262579617/Put/seqid=0 2024-11-10T18:16:20,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742366_1542 (size=12304) 2024-11-10T18:16:20,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:20,942 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110dcb6bc304d794a16821386420e9022b8_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcb6bc304d794a16821386420e9022b8_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:20,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5201bb957741454bba815900515fb8cd, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:20,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5201bb957741454bba815900515fb8cd is 175, key is test_row_0/A:col10/1731262579617/Put/seqid=0 2024-11-10T18:16:20,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742367_1543 (size=31105) 2024-11-10T18:16:20,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T18:16:21,349 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=149, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5201bb957741454bba815900515fb8cd 2024-11-10T18:16:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/20be7750453f41d9b2782be3c6130903 is 50, key is test_row_0/B:col10/1731262579617/Put/seqid=0 2024-11-10T18:16:21,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742368_1544 (size=12151) 2024-11-10T18:16:21,359 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/20be7750453f41d9b2782be3c6130903 2024-11-10T18:16:21,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/67efaeb744fc46ef86414d2df17d6869 is 50, key is test_row_0/C:col10/1731262579617/Put/seqid=0 2024-11-10T18:16:21,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742369_1545 (size=12151) 2024-11-10T18:16:21,369 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/67efaeb744fc46ef86414d2df17d6869 2024-11-10T18:16:21,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5201bb957741454bba815900515fb8cd as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd 2024-11-10T18:16:21,377 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd, entries=150, sequenceid=149, filesize=30.4 K 2024-11-10T18:16:21,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/20be7750453f41d9b2782be3c6130903 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/20be7750453f41d9b2782be3c6130903 2024-11-10T18:16:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,380 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/20be7750453f41d9b2782be3c6130903, entries=150, sequenceid=149, filesize=11.9 K 2024-11-10T18:16:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/67efaeb744fc46ef86414d2df17d6869 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/67efaeb744fc46ef86414d2df17d6869 2024-11-10T18:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,385 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/67efaeb744fc46ef86414d2df17d6869, entries=150, sequenceid=149, filesize=11.9 K 2024-11-10T18:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,385 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 01f94435f1c5387043f08668493d3e9e in 456ms, sequenceid=149, compaction requested=true 2024-11-10T18:16:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:21,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-10T18:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-10T18:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-10T18:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,389 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5290 sec 2024-11-10T18:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.5340 sec 2024-11-10T18:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:21,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-10T18:16:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:21,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:21,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba1f15dfe2e047f79c1bc904f0900624_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:21,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742370_1546 (size=14794) 2024-11-10T18:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731262581839, max timestamp=1731262581854Num uniques 859; 2024-11-10T18:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731262581839, max timestamp=1731262581854Num uniques 859; 2024-11-10T18:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:16:21,857 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731262581839, max timestamp=1731262581854Num uniques 859; 2024-11-10T18:16:21,858 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/691ea56905d2:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-10T18:16:21,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:21,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262641895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:21,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262641897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:21,904 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:21,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262641898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:21,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:21,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-10T18:16:22,005 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:22,006 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:22,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-10T18:16:22,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:22,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:22,008 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:22,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:22,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262642005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262642014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262642014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:22,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262642214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262642224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262642224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:22,239 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba1f15dfe2e047f79c1bc904f0900624_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba1f15dfe2e047f79c1bc904f0900624_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:22,240 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5a29107d3bfc402fbcbe8213d2f88688, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:22,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5a29107d3bfc402fbcbe8213d2f88688 is 175, key is test_row_0/A:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:22,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742371_1547 (size=39749) 2024-11-10T18:16:22,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:22,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:22,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:22,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262642454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,459 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262642475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,480 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262642524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,530 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262642534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262642534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:22,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:22,644 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5a29107d3bfc402fbcbe8213d2f88688 2024-11-10T18:16:22,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/ee3811b1041b4755b466ba95140dac6b is 50, key is test_row_0/B:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:22,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742372_1548 (size=12151) 2024-11-10T18:16:22,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:22,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262643044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262643044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:23,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:23,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262643045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:23,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:23,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/ee3811b1041b4755b466ba95140dac6b 2024-11-10T18:16:23,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a6111ca3117446ca918678824ad7b65d is 50, key is test_row_0/C:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:23,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742373_1549 (size=12151) 2024-11-10T18:16:23,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:23,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:23,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:23,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:23,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a6111ca3117446ca918678824ad7b65d 2024-11-10T18:16:23,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5a29107d3bfc402fbcbe8213d2f88688 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688 2024-11-10T18:16:23,472 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688, entries=200, sequenceid=159, filesize=38.8 K 2024-11-10T18:16:23,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/ee3811b1041b4755b466ba95140dac6b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ee3811b1041b4755b466ba95140dac6b 2024-11-10T18:16:23,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ee3811b1041b4755b466ba95140dac6b, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T18:16:23,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a6111ca3117446ca918678824ad7b65d as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6111ca3117446ca918678824ad7b65d 2024-11-10T18:16:23,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6111ca3117446ca918678824ad7b65d, entries=150, sequenceid=159, filesize=11.9 K 2024-11-10T18:16:23,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 01f94435f1c5387043f08668493d3e9e in 1666ms, sequenceid=159, compaction requested=true 2024-11-10T18:16:23,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:23,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:23,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:23,481 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:16:23,481 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:16:23,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:23,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:23,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:23,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:23,483 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141714 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:16:23,483 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:16:23,483 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:23,483 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:23,483 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,483 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,483 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/f234d3f2d5854b78bba984854cb60b33, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=138.4 K 2024-11-10T18:16:23,483 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/16ab32ae684148ff8b8f4133e44b06fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/59244f0ac39143edbb7451b6677ae739, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/20be7750453f41d9b2782be3c6130903, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ee3811b1041b4755b466ba95140dac6b] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=47.5 K 2024-11-10T18:16:23,483 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,483 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/f234d3f2d5854b78bba984854cb60b33, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688] 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 16ab32ae684148ff8b8f4133e44b06fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731262577598 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting f234d3f2d5854b78bba984854cb60b33, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731262577598 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 59244f0ac39143edbb7451b6677ae739, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262578282 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeb7e3aad68d493387ed8b4e5f09c1b9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262578267 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 20be7750453f41d9b2782be3c6130903, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731262579617 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5201bb957741454bba815900515fb8cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731262579617 2024-11-10T18:16:23,484 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ee3811b1041b4755b466ba95140dac6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731262581807 2024-11-10T18:16:23,485 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a29107d3bfc402fbcbe8213d2f88688, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731262581807 2024-11-10T18:16:23,491 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:23,493 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:23,493 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/2ca3f08e05b54ad4b9a350ff1a654a56 is 50, key is test_row_0/B:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:23,495 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110ba2e417e0b4f482793b978c2722ff28d_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:23,497 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110ba2e417e0b4f482793b978c2722ff28d_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:23,498 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ba2e417e0b4f482793b978c2722ff28d_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742374_1550 (size=12493) 2024-11-10T18:16:23,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742375_1551 (size=4469) 2024-11-10T18:16:23,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-10T18:16:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,536 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-10T18:16:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-10T18:16:23,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d25de3fbda1e46f9a73ed187e3e661fa_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262581896/Put/seqid=0 2024-11-10T18:16:23,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742376_1552 (size=12304) 2024-11-10T18:16:23,907 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/2ca3f08e05b54ad4b9a350ff1a654a56 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/2ca3f08e05b54ad4b9a350ff1a654a56 2024-11-10T18:16:23,911 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into 2ca3f08e05b54ad4b9a350ff1a654a56(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:23,911 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:23,911 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=12, startTime=1731262583481; duration=0sec 2024-11-10T18:16:23,911 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:23,911 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:23,911 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#466 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:23,911 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-10T18:16:23,912 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/0ce98ebe66d148288a55fd09dbcc021f is 175, key is test_row_0/A:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:23,913 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-10T18:16:23,913 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:23,913 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:23,913 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/dfa21e1cf9b04a56b2dcf403407ed39f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5b9612c7508349dda360ee1ecee1a034, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/67efaeb744fc46ef86414d2df17d6869, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6111ca3117446ca918678824ad7b65d] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=47.5 K 2024-11-10T18:16:23,914 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting dfa21e1cf9b04a56b2dcf403407ed39f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731262577598 2024-11-10T18:16:23,914 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b9612c7508349dda360ee1ecee1a034, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731262578282 2024-11-10T18:16:23,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742377_1553 (size=31447) 2024-11-10T18:16:23,915 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 67efaeb744fc46ef86414d2df17d6869, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731262579617 2024-11-10T18:16:23,915 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a6111ca3117446ca918678824ad7b65d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731262581807 2024-11-10T18:16:23,918 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/0ce98ebe66d148288a55fd09dbcc021f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0ce98ebe66d148288a55fd09dbcc021f 2024-11-10T18:16:23,924 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into 0ce98ebe66d148288a55fd09dbcc021f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:23,924 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:23,924 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=12, startTime=1731262583481; duration=0sec 2024-11-10T18:16:23,924 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:23,924 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:23,924 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:23,924 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/fe12d3df72364d6b93578131ce352b5c is 50, key is test_row_0/C:col10/1731262581814/Put/seqid=0 2024-11-10T18:16:23,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742378_1554 (size=12493) 2024-11-10T18:16:23,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:23,950 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d25de3fbda1e46f9a73ed187e3e661fa_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d25de3fbda1e46f9a73ed187e3e661fa_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/d23370e1f3fb4cbd8eea111d6a08f9e8, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:23,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/d23370e1f3fb4cbd8eea111d6a08f9e8 is 175, key is test_row_0/A:col10/1731262581896/Put/seqid=0 2024-11-10T18:16:23,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742379_1555 (size=31105) 2024-11-10T18:16:24,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:24,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262644076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,081 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262644077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262644077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:24,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262644184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,189 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262644194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262644194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,332 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/fe12d3df72364d6b93578131ce352b5c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe12d3df72364d6b93578131ce352b5c 2024-11-10T18:16:24,336 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into fe12d3df72364d6b93578131ce352b5c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:24,336 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:24,336 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=12, startTime=1731262583482; duration=0sec 2024-11-10T18:16:24,336 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:24,336 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:24,355 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/d23370e1f3fb4cbd8eea111d6a08f9e8 2024-11-10T18:16:24,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/0ba05f7a80e5430e9300462936ed28e4 is 50, key is test_row_0/B:col10/1731262581896/Put/seqid=0 2024-11-10T18:16:24,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742380_1556 (size=12151) 2024-11-10T18:16:24,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262644395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,400 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262644404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262644404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262644705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,709 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262644715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262644715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:24,765 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/0ba05f7a80e5430e9300462936ed28e4 2024-11-10T18:16:24,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/5215df632fe341078c86d6a153bdccef is 50, key is test_row_0/C:col10/1731262581896/Put/seqid=0 2024-11-10T18:16:24,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742381_1557 (size=12151) 2024-11-10T18:16:25,176 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/5215df632fe341078c86d6a153bdccef 2024-11-10T18:16:25,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/d23370e1f3fb4cbd8eea111d6a08f9e8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8 2024-11-10T18:16:25,183 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8, entries=150, sequenceid=195, filesize=30.4 K 2024-11-10T18:16:25,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/0ba05f7a80e5430e9300462936ed28e4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/0ba05f7a80e5430e9300462936ed28e4 2024-11-10T18:16:25,187 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/0ba05f7a80e5430e9300462936ed28e4, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T18:16:25,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/5215df632fe341078c86d6a153bdccef as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5215df632fe341078c86d6a153bdccef 2024-11-10T18:16:25,191 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5215df632fe341078c86d6a153bdccef, entries=150, sequenceid=195, filesize=11.9 K 2024-11-10T18:16:25,192 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 01f94435f1c5387043f08668493d3e9e in 1656ms, sequenceid=195, compaction requested=false 2024-11-10T18:16:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:25,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-10T18:16:25,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-10T18:16:25,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-10T18:16:25,195 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1840 sec 2024-11-10T18:16:25,198 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.1890 sec 2024-11-10T18:16:25,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:25,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-10T18:16:25,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:25,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:25,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:25,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:25,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:25,217 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:25,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106037065274da41cdaea0bd88a485e167_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:25,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742382_1558 (size=14794) 2024-11-10T18:16:25,229 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:25,233 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411106037065274da41cdaea0bd88a485e167_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106037065274da41cdaea0bd88a485e167_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:25,233 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/ed7c44c4b880471b91cd644da5767359, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:25,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/ed7c44c4b880471b91cd644da5767359 is 175, key is test_row_0/A:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:25,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742383_1559 (size=39749) 2024-11-10T18:16:25,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262645251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262645254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262645255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262645365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262645365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262645365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262645575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262645575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262645575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,647 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/ed7c44c4b880471b91cd644da5767359 2024-11-10T18:16:25,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/d52f3a470dbd4416a1ffaa92b4574093 is 50, key is test_row_0/B:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:25,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742384_1560 (size=12151) 2024-11-10T18:16:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262645885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262645885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262645885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:25,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/d52f3a470dbd4416a1ffaa92b4574093 2024-11-10T18:16:26,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/d1b25c0d007d48508500f0c62e584c52 is 50, key is test_row_0/C:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:26,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742385_1561 (size=12151) 2024-11-10T18:16:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-10T18:16:26,175 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:26,175 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-10T18:16:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T18:16:26,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:26,177 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:26,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T18:16:26,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T18:16:26,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:26,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,330 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:26,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:26,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262646394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262646394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262646394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,467 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/d1b25c0d007d48508500f0c62e584c52 2024-11-10T18:16:26,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/ed7c44c4b880471b91cd644da5767359 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359 2024-11-10T18:16:26,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359, entries=200, sequenceid=210, filesize=38.8 K 2024-11-10T18:16:26,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/d52f3a470dbd4416a1ffaa92b4574093 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d52f3a470dbd4416a1ffaa92b4574093 2024-11-10T18:16:26,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60432 deadline: 1731262646474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,477 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_0' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d52f3a470dbd4416a1ffaa92b4574093, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T18:16:26,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/d1b25c0d007d48508500f0c62e584c52 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/d1b25c0d007d48508500f0c62e584c52 2024-11-10T18:16:26,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/d1b25c0d007d48508500f0c62e584c52, entries=150, sequenceid=210, filesize=11.9 K 2024-11-10T18:16:26,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 01f94435f1c5387043f08668493d3e9e in 1265ms, sequenceid=210, compaction requested=true 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:26,482 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:26,482 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:26,482 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:26,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:26,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,483 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-10T18:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:26,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:26,485 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:26,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102301 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:26,485 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:26,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:26,485 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,485 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,486 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/2ca3f08e05b54ad4b9a350ff1a654a56, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/0ba05f7a80e5430e9300462936ed28e4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d52f3a470dbd4416a1ffaa92b4574093] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=35.9 K 2024-11-10T18:16:26,486 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0ce98ebe66d148288a55fd09dbcc021f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=99.9 K 2024-11-10T18:16:26,486 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,486 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0ce98ebe66d148288a55fd09dbcc021f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359] 2024-11-10T18:16:26,486 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce98ebe66d148288a55fd09dbcc021f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731262581807 2024-11-10T18:16:26,486 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ca3f08e05b54ad4b9a350ff1a654a56, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731262581807 2024-11-10T18:16:26,486 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ba05f7a80e5430e9300462936ed28e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262581878 2024-11-10T18:16:26,487 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d23370e1f3fb4cbd8eea111d6a08f9e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262581878 2024-11-10T18:16:26,487 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d52f3a470dbd4416a1ffaa92b4574093, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731262584068 2024-11-10T18:16:26,487 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting ed7c44c4b880471b91cd644da5767359, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731262584068 2024-11-10T18:16:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T18:16:26,500 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:26,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111070590cfb00be443c90e0cbf935ecf969_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262585243/Put/seqid=0 2024-11-10T18:16:26,501 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:26,501 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/d6dc255340914952aacea24fd2477341 is 50, key is test_row_0/B:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:26,502 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110b8e17d9cb1754f0690b6a3c5a48f6fad_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:26,504 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110b8e17d9cb1754f0690b6a3c5a48f6fad_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:26,505 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b8e17d9cb1754f0690b6a3c5a48f6fad_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:26,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742387_1563 (size=12595) 2024-11-10T18:16:26,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742386_1562 (size=12304) 2024-11-10T18:16:26,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742388_1564 (size=4469) 2024-11-10T18:16:26,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:26,516 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#476 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:26,516 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/78e3289c41374176923835c38e46a828 is 175, key is test_row_0/A:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:26,519 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111070590cfb00be443c90e0cbf935ecf969_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111070590cfb00be443c90e0cbf935ecf969_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:26,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7de840af1ab64600b28f70a8261da945, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:26,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7de840af1ab64600b28f70a8261da945 is 175, key is test_row_0/A:col10/1731262585243/Put/seqid=0 2024-11-10T18:16:26,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742389_1565 (size=31549) 2024-11-10T18:16:26,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742390_1566 (size=31105) 2024-11-10T18:16:26,528 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/78e3289c41374176923835c38e46a828 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/78e3289c41374176923835c38e46a828 2024-11-10T18:16:26,528 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7de840af1ab64600b28f70a8261da945 2024-11-10T18:16:26,534 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into 78e3289c41374176923835c38e46a828(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:26,534 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:26,534 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=13, startTime=1731262586482; duration=0sec 2024-11-10T18:16:26,535 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:26,535 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:26,535 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:26,535 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:26,536 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:26,536 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,536 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe12d3df72364d6b93578131ce352b5c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5215df632fe341078c86d6a153bdccef, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/d1b25c0d007d48508500f0c62e584c52] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=35.9 K 2024-11-10T18:16:26,537 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting fe12d3df72364d6b93578131ce352b5c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1731262581807 2024-11-10T18:16:26,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/40b2cfd22e5c4d5bb8766b1965a86360 is 50, key is test_row_0/B:col10/1731262585243/Put/seqid=0 2024-11-10T18:16:26,537 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 5215df632fe341078c86d6a153bdccef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731262581878 2024-11-10T18:16:26,538 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d1b25c0d007d48508500f0c62e584c52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731262584068 2024-11-10T18:16:26,547 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:26,547 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/9068fc0e4e064c33bc78211c34b4fbb8 is 50, key is test_row_0/C:col10/1731262584076/Put/seqid=0 2024-11-10T18:16:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262646550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742391_1567 (size=12151) 2024-11-10T18:16:26,558 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/40b2cfd22e5c4d5bb8766b1965a86360 2024-11-10T18:16:26,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/9e0766e48f944969945b217c23463a6e is 50, key is test_row_0/C:col10/1731262585243/Put/seqid=0 2024-11-10T18:16:26,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742392_1568 (size=12595) 2024-11-10T18:16:26,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742393_1569 (size=12151) 2024-11-10T18:16:26,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262646664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T18:16:26,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:26,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262646874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:26,876 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:26,913 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/d6dc255340914952aacea24fd2477341 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d6dc255340914952aacea24fd2477341 2024-11-10T18:16:26,917 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into d6dc255340914952aacea24fd2477341(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:26,917 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:26,917 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=13, startTime=1731262586482; duration=0sec 2024-11-10T18:16:26,917 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:26,917 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:26,977 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/9068fc0e4e064c33bc78211c34b4fbb8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9068fc0e4e064c33bc78211c34b4fbb8 2024-11-10T18:16:26,979 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/9e0766e48f944969945b217c23463a6e 2024-11-10T18:16:26,981 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into 9068fc0e4e064c33bc78211c34b4fbb8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:26,981 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:26,981 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=13, startTime=1731262586482; duration=0sec 2024-11-10T18:16:26,981 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:26,981 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:26,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7de840af1ab64600b28f70a8261da945 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945 2024-11-10T18:16:26,985 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945, entries=150, sequenceid=230, filesize=30.4 K 2024-11-10T18:16:26,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/40b2cfd22e5c4d5bb8766b1965a86360 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/40b2cfd22e5c4d5bb8766b1965a86360 2024-11-10T18:16:26,989 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/40b2cfd22e5c4d5bb8766b1965a86360, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T18:16:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/9e0766e48f944969945b217c23463a6e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9e0766e48f944969945b217c23463a6e 2024-11-10T18:16:26,993 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9e0766e48f944969945b217c23463a6e, entries=150, sequenceid=230, filesize=11.9 K 2024-11-10T18:16:26,993 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 01f94435f1c5387043f08668493d3e9e in 510ms, sequenceid=230, compaction requested=false 2024-11-10T18:16:26,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:26,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:26,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-10T18:16:26,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-10T18:16:26,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-10T18:16:26,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 817 msec 2024-11-10T18:16:26,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 822 msec 2024-11-10T18:16:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:27,186 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-10T18:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:27,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:27,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ead927c83fe54251a55fd9ba0a74e5ff_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742394_1570 (size=12304) 2024-11-10T18:16:27,196 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:27,199 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110ead927c83fe54251a55fd9ba0a74e5ff_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ead927c83fe54251a55fd9ba0a74e5ff_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:27,200 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5ceecf3d5175493291cb93f1b1f4ca9e, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:27,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5ceecf3d5175493291cb93f1b1f4ca9e is 175, key is test_row_0/A:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:27,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742395_1571 (size=31105) 2024-11-10T18:16:27,204 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5ceecf3d5175493291cb93f1b1f4ca9e 2024-11-10T18:16:27,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/55f9644e5269456c9780adf6f60876c7 is 50, key is test_row_0/B:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:27,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742396_1572 (size=12151) 2024-11-10T18:16:27,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262647265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-10T18:16:27,315 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:27,315 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-10T18:16:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T18:16:27,316 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:27,317 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:27,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:27,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262647375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262647404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,409 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262647415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262647415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,419 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:27,419 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,420 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T18:16:27,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T18:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:27,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:27,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:27,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262647584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/55f9644e5269456c9780adf6f60876c7 2024-11-10T18:16:27,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a5897bbf299d4000b35709201ef57aee is 50, key is test_row_0/C:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:27,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T18:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,622 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:27,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:27,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742397_1573 (size=12151) 2024-11-10T18:16:27,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a5897bbf299d4000b35709201ef57aee 2024-11-10T18:16:27,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/5ceecf3d5175493291cb93f1b1f4ca9e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e 2024-11-10T18:16:27,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e, entries=150, sequenceid=246, filesize=30.4 K 2024-11-10T18:16:27,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/55f9644e5269456c9780adf6f60876c7 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/55f9644e5269456c9780adf6f60876c7 2024-11-10T18:16:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T18:16:27,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/55f9644e5269456c9780adf6f60876c7, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T18:16:27,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a5897bbf299d4000b35709201ef57aee as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a5897bbf299d4000b35709201ef57aee 2024-11-10T18:16:27,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a5897bbf299d4000b35709201ef57aee, entries=150, sequenceid=246, filesize=11.9 K 2024-11-10T18:16:27,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 01f94435f1c5387043f08668493d3e9e in 455ms, sequenceid=246, compaction requested=true 2024-11-10T18:16:27,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:27,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:27,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:27,641 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:27,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:27,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:27,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:27,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:27,642 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:27,642 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93759 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:27,642 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:27,643 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,643 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/78e3289c41374176923835c38e46a828, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=91.6 K 2024-11-10T18:16:27,643 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,643 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/78e3289c41374176923835c38e46a828, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e] 2024-11-10T18:16:27,643 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:27,644 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:27,644 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,644 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d6dc255340914952aacea24fd2477341, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/40b2cfd22e5c4d5bb8766b1965a86360, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/55f9644e5269456c9780adf6f60876c7] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=36.0 K 2024-11-10T18:16:27,644 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78e3289c41374176923835c38e46a828, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731262584068 2024-11-10T18:16:27,645 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting d6dc255340914952aacea24fd2477341, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731262584068 2024-11-10T18:16:27,645 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7de840af1ab64600b28f70a8261da945, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731262585243 2024-11-10T18:16:27,645 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 40b2cfd22e5c4d5bb8766b1965a86360, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731262585243 2024-11-10T18:16:27,645 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ceecf3d5175493291cb93f1b1f4ca9e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262586542 2024-11-10T18:16:27,645 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 55f9644e5269456c9780adf6f60876c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262586542 2024-11-10T18:16:27,654 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:27,654 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#483 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:27,655 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/136a035799a5404aa7ad13e82c45231b is 50, key is test_row_0/B:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:27,655 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110b6ebdb8ef2a942d3affb8184b793a5f4_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:27,658 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110b6ebdb8ef2a942d3affb8184b793a5f4_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:27,659 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110b6ebdb8ef2a942d3affb8184b793a5f4_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:27,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742398_1574 (size=12697) 2024-11-10T18:16:27,676 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/136a035799a5404aa7ad13e82c45231b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/136a035799a5404aa7ad13e82c45231b 2024-11-10T18:16:27,680 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into 136a035799a5404aa7ad13e82c45231b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:27,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:27,680 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=13, startTime=1731262587641; duration=0sec 2024-11-10T18:16:27,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:27,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:27,680 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:27,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:27,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:27,685 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,685 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9068fc0e4e064c33bc78211c34b4fbb8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9e0766e48f944969945b217c23463a6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a5897bbf299d4000b35709201ef57aee] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=36.0 K 2024-11-10T18:16:27,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 9068fc0e4e064c33bc78211c34b4fbb8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731262584068 2024-11-10T18:16:27,685 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e0766e48f944969945b217c23463a6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731262585243 2024-11-10T18:16:27,686 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a5897bbf299d4000b35709201ef57aee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262586542 2024-11-10T18:16:27,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742399_1575 (size=4469) 2024-11-10T18:16:27,693 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:27,693 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/fe52041968de428dae71f6250c17069f is 50, key is test_row_0/C:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742400_1576 (size=12697) 2024-11-10T18:16:27,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:27,775 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:27,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:27,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100a05fcbba9b64015a3f5d631dc07def6_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262587264/Put/seqid=0 2024-11-10T18:16:27,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742401_1577 (size=12454) 2024-11-10T18:16:27,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:27,790 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411100a05fcbba9b64015a3f5d631dc07def6_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100a05fcbba9b64015a3f5d631dc07def6_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:27,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/387149e12b9d48fab02177d69b64838e, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:27,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/387149e12b9d48fab02177d69b64838e is 175, key is test_row_0/A:col10/1731262587264/Put/seqid=0 2024-11-10T18:16:27,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742402_1578 (size=31255) 2024-11-10T18:16:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:27,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:27,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262647938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:27,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:27,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T18:16:28,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:28,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262648045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:28,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:28,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:28,093 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:28,094 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/fc0d51f6b0a5416e96388826da8ad545 is 175, key is test_row_0/A:col10/1731262587185/Put/seqid=0 2024-11-10T18:16:28,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742403_1579 (size=31651) 2024-11-10T18:16:28,102 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/fe52041968de428dae71f6250c17069f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe52041968de428dae71f6250c17069f 2024-11-10T18:16:28,105 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into fe52041968de428dae71f6250c17069f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:28,105 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:28,105 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=13, startTime=1731262587642; duration=0sec 2024-11-10T18:16:28,106 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:28,106 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:28,196 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/387149e12b9d48fab02177d69b64838e 2024-11-10T18:16:28,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/3531d866e1594513acfd68d105852521 is 50, key is test_row_0/B:col10/1731262587264/Put/seqid=0 2024-11-10T18:16:28,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742404_1580 (size=12301) 2024-11-10T18:16:28,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262648254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:28,257 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:28,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T18:16:28,501 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/fc0d51f6b0a5416e96388826da8ad545 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/fc0d51f6b0a5416e96388826da8ad545 2024-11-10T18:16:28,505 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into fc0d51f6b0a5416e96388826da8ad545(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:28,505 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:28,505 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=13, startTime=1731262587641; duration=0sec 2024-11-10T18:16:28,505 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:28,505 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:28,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:28,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262648565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:28,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:28,607 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/3531d866e1594513acfd68d105852521 2024-11-10T18:16:28,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/57a661d374364a81a8fe11f0087dfa6e is 50, key is test_row_0/C:col10/1731262587264/Put/seqid=0 2024-11-10T18:16:28,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742405_1581 (size=12301) 2024-11-10T18:16:29,017 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/57a661d374364a81a8fe11f0087dfa6e 2024-11-10T18:16:29,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/387149e12b9d48fab02177d69b64838e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e 2024-11-10T18:16:29,024 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e, entries=150, sequenceid=266, filesize=30.5 K 2024-11-10T18:16:29,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/3531d866e1594513acfd68d105852521 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/3531d866e1594513acfd68d105852521 2024-11-10T18:16:29,028 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/3531d866e1594513acfd68d105852521, entries=150, sequenceid=266, filesize=12.0 K 2024-11-10T18:16:29,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/57a661d374364a81a8fe11f0087dfa6e as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/57a661d374364a81a8fe11f0087dfa6e 2024-11-10T18:16:29,032 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/57a661d374364a81a8fe11f0087dfa6e, entries=150, sequenceid=266, filesize=12.0 K 2024-11-10T18:16:29,033 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 01f94435f1c5387043f08668493d3e9e in 1258ms, sequenceid=266, compaction requested=false 2024-11-10T18:16:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-10T18:16:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-10T18:16:29,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-10T18:16:29,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7170 sec 2024-11-10T18:16:29,037 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.7210 sec 2024-11-10T18:16:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:29,079 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-10T18:16:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:29,079 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:29,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c2fc57294e5483d9f2661d376be224a_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742406_1582 (size=14994) 2024-11-10T18:16:29,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262649152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262649264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262649425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262649425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,427 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,427 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262649444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,445 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,446 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-10T18:16:29,465 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:29,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-10T18:16:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-10T18:16:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:29,467 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-10T18:16:29,468 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-10T18:16:29,468 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-10T18:16:29,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262649475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,492 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:29,495 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411104c2fc57294e5483d9f2661d376be224a_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c2fc57294e5483d9f2661d376be224a_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:29,496 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/0aee7a76539e487093c700ebdb27e8cf, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:29,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/0aee7a76539e487093c700ebdb27e8cf is 175, key is test_row_0/A:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:29,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742407_1583 (size=39949) 2024-11-10T18:16:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:29,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262649784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:29,900 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=282, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/0aee7a76539e487093c700ebdb27e8cf 2024-11-10T18:16:29,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/7cb0b122f14f43ecbb9ac51ceb0744f2 is 50, key is test_row_0/B:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:29,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742408_1584 (size=12301) 2024-11-10T18:16:29,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:29,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:29,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:30,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:30,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262650294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:30,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:30,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/7cb0b122f14f43ecbb9ac51ceb0744f2 2024-11-10T18:16:30,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/cf1db2ae6e7f430491654dafb1685b68 is 50, key is test_row_0/C:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:30,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742409_1585 (size=12301) 2024-11-10T18:16:30,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:30,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:30,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T18:16:30,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/cf1db2ae6e7f430491654dafb1685b68 2024-11-10T18:16:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/0aee7a76539e487093c700ebdb27e8cf as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf 2024-11-10T18:16:30,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf, entries=200, sequenceid=282, filesize=39.0 K 2024-11-10T18:16:30,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/7cb0b122f14f43ecbb9ac51ceb0744f2 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7cb0b122f14f43ecbb9ac51ceb0744f2 2024-11-10T18:16:30,743 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7cb0b122f14f43ecbb9ac51ceb0744f2, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T18:16:30,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/cf1db2ae6e7f430491654dafb1685b68 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/cf1db2ae6e7f430491654dafb1685b68 2024-11-10T18:16:30,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/cf1db2ae6e7f430491654dafb1685b68, entries=150, sequenceid=282, filesize=12.0 K 2024-11-10T18:16:30,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 01f94435f1c5387043f08668493d3e9e in 1669ms, sequenceid=282, compaction requested=true 2024-11-10T18:16:30,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:30,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:30,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:30,748 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:30,748 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:30,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:30,748 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:30,748 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:30,748 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:30,748 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:30,749 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,749 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,749 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/136a035799a5404aa7ad13e82c45231b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/3531d866e1594513acfd68d105852521, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7cb0b122f14f43ecbb9ac51ceb0744f2] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=36.4 K 2024-11-10T18:16:30,749 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/fc0d51f6b0a5416e96388826da8ad545, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=100.4 K 2024-11-10T18:16:30,749 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,749 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/fc0d51f6b0a5416e96388826da8ad545, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf] 2024-11-10T18:16:30,749 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 136a035799a5404aa7ad13e82c45231b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262586542 2024-11-10T18:16:30,749 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc0d51f6b0a5416e96388826da8ad545, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262586542 2024-11-10T18:16:30,749 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 3531d866e1594513acfd68d105852521, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731262587253 2024-11-10T18:16:30,749 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 387149e12b9d48fab02177d69b64838e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731262587253 2024-11-10T18:16:30,750 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 7cb0b122f14f43ecbb9ac51ceb0744f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262587933 2024-11-10T18:16:30,750 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aee7a76539e487093c700ebdb27e8cf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262587927 2024-11-10T18:16:30,755 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:30,756 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110448a59bcd2a642b5ad567d5e206505f0_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:30,758 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#493 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:30,758 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/94b5a9c62a174edf9e190021c1364272 is 50, key is test_row_0/B:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:30,759 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110448a59bcd2a642b5ad567d5e206505f0_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:30,759 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110448a59bcd2a642b5ad567d5e206505f0_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:30,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742411_1587 (size=12949) 2024-11-10T18:16:30,783 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/94b5a9c62a174edf9e190021c1364272 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/94b5a9c62a174edf9e190021c1364272 2024-11-10T18:16:30,788 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into 94b5a9c62a174edf9e190021c1364272(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:30,788 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:30,788 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=13, startTime=1731262590747; duration=0sec 2024-11-10T18:16:30,788 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:30,788 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:30,788 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:30,790 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:30,790 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:30,790 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,790 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe52041968de428dae71f6250c17069f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/57a661d374364a81a8fe11f0087dfa6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/cf1db2ae6e7f430491654dafb1685b68] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=36.4 K 2024-11-10T18:16:30,791 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting fe52041968de428dae71f6250c17069f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731262586542 2024-11-10T18:16:30,791 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 57a661d374364a81a8fe11f0087dfa6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1731262587253 2024-11-10T18:16:30,791 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting cf1db2ae6e7f430491654dafb1685b68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262587933 2024-11-10T18:16:30,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742410_1586 (size=4469) 2024-11-10T18:16:30,797 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#492 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:30,797 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/d064571f9be74d1c8b3da48ac2a1996c is 175, key is test_row_0/A:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:30,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742412_1588 (size=31903) 2024-11-10T18:16:30,803 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:30,803 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/da264a9e2e4f4e31860bdd831c24d3d9 is 50, key is test_row_0/C:col10/1731262587933/Put/seqid=0 2024-11-10T18:16:30,806 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/d064571f9be74d1c8b3da48ac2a1996c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d064571f9be74d1c8b3da48ac2a1996c 2024-11-10T18:16:30,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742413_1589 (size=12949) 2024-11-10T18:16:30,811 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into d064571f9be74d1c8b3da48ac2a1996c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:30,811 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:30,811 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=13, startTime=1731262590747; duration=0sec 2024-11-10T18:16:30,811 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:30,811 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:30,817 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/da264a9e2e4f4e31860bdd831c24d3d9 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/da264a9e2e4f4e31860bdd831c24d3d9 2024-11-10T18:16:30,823 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into da264a9e2e4f4e31860bdd831c24d3d9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:30,823 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:30,823 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=13, startTime=1731262590748; duration=0sec 2024-11-10T18:16:30,823 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:30,823 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:30,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=32909 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-10T18:16:30,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:30,843 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T18:16:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d6ec5d2d4d434f03a642a4067930ccdd_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262589138/Put/seqid=0 2024-11-10T18:16:30,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742414_1590 (size=12454) 2024-11-10T18:16:31,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:31,262 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110d6ec5d2d4d434f03a642a4067930ccdd_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d6ec5d2d4d434f03a642a4067930ccdd_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:31,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/34e747766e2147339ad8b4c67b98de2a, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:31,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/34e747766e2147339ad8b4c67b98de2a is 175, key is test_row_0/A:col10/1731262589138/Put/seqid=0 2024-11-10T18:16:31,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742415_1591 (size=31255) 2024-11-10T18:16:31,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:31,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. as already flushing 2024-11-10T18:16:31,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:31,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262651358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:31,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:31,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262651465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:31,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:31,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:31,668 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=303, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/34e747766e2147339ad8b4c67b98de2a 2024-11-10T18:16:31,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/c84fbfadc31c4e62a60ebcb29782092a is 50, key is test_row_0/B:col10/1731262589138/Put/seqid=0 2024-11-10T18:16:31,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742416_1592 (size=12301) 2024-11-10T18:16:31,677 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/c84fbfadc31c4e62a60ebcb29782092a 2024-11-10T18:16:31,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:31,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262651675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:31,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/81446cb8f54c461aaa344b8e74a4dd04 is 50, key is test_row_0/C:col10/1731262589138/Put/seqid=0 2024-11-10T18:16:31,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742417_1593 (size=12301) 2024-11-10T18:16:31,707 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/81446cb8f54c461aaa344b8e74a4dd04 2024-11-10T18:16:31,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/34e747766e2147339ad8b4c67b98de2a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a 2024-11-10T18:16:31,715 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a, entries=150, sequenceid=303, filesize=30.5 K 2024-11-10T18:16:31,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/c84fbfadc31c4e62a60ebcb29782092a as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/c84fbfadc31c4e62a60ebcb29782092a 2024-11-10T18:16:31,719 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/c84fbfadc31c4e62a60ebcb29782092a, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T18:16:31,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/81446cb8f54c461aaa344b8e74a4dd04 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/81446cb8f54c461aaa344b8e74a4dd04 2024-11-10T18:16:31,724 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/81446cb8f54c461aaa344b8e74a4dd04, entries=150, sequenceid=303, filesize=12.0 K 2024-11-10T18:16:31,725 INFO [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 01f94435f1c5387043f08668493d3e9e in 883ms, sequenceid=303, compaction requested=false 2024-11-10T18:16:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:31,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/691ea56905d2:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-10T18:16:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-10T18:16:31,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-10T18:16:31,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2590 sec 2024-11-10T18:16:31,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.2660 sec 2024-11-10T18:16:31,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:31,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-10T18:16:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:31,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:31,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a54a66c706ee4449af9f7826c5762d6c_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:31,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742418_1594 (size=12454) 2024-11-10T18:16:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262652048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:32,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:32,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262652154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:32,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262652365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:32,398 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:32,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110a54a66c706ee4449af9f7826c5762d6c_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a54a66c706ee4449af9f7826c5762d6c_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:32,402 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7dc3f8b9b36d42c7a5652a9c88deddf6, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:32,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7dc3f8b9b36d42c7a5652a9c88deddf6 is 175, key is test_row_0/A:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:32,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742419_1595 (size=31255) 2024-11-10T18:16:32,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:32,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262652674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:32,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:32,806 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7dc3f8b9b36d42c7a5652a9c88deddf6 2024-11-10T18:16:32,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/8d33d209163d4660bd3ddac2b3e53276 is 50, key is test_row_0/B:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:32,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742420_1596 (size=12301) 2024-11-10T18:16:33,186 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:33,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60402 deadline: 1731262653185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:33,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/8d33d209163d4660bd3ddac2b3e53276 2024-11-10T18:16:33,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a6fcd18af1c24409a09f56076e5dbf70 is 50, key is test_row_0/C:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:33,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742421_1597 (size=12301) 2024-11-10T18:16:33,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:33,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60456 deadline: 1731262653435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:33,439 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,439 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60396 deadline: 1731262653454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:33,458 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_1' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-10T18:16:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=32909 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60452 deadline: 1731262653464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 2024-11-10T18:16:33,468 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 691ea56905d2,32909,1731262402364 for 'test_row_2' in 01f94435f1c5387043f08668493d3e9e of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-10T18:16:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 , the old value is region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-10T18:16:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., hostname=691ea56905d2,32909,1731262402364, seqNum=5 because the exception is null or not the one we care about 2024-11-10T18:16:33,558 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-10T18:16:33,558 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:33,558 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,558 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,558 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:33,559 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-10T18:16:33,559 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:33,559 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,559 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:33,561 INFO [Thread-2312 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2312. 2024-11-10T18:16:33,561 DEBUG [Thread-2312 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:33,561 DEBUG [Thread-2312 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,561 DEBUG [Thread-2312 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,561 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:33,562 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-10T18:16:33,562 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:33,562 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,562 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,562 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:33,564 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-10T18:16:33,564 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:33,564 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,564 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:33,564 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:33,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-10T18:16:33,625 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:33,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a6fcd18af1c24409a09f56076e5dbf70 2024-11-10T18:16:33,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/7dc3f8b9b36d42c7a5652a9c88deddf6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6 2024-11-10T18:16:33,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6, entries=150, sequenceid=318, filesize=30.5 K 2024-11-10T18:16:33,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/8d33d209163d4660bd3ddac2b3e53276 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/8d33d209163d4660bd3ddac2b3e53276 2024-11-10T18:16:33,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/8d33d209163d4660bd3ddac2b3e53276, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T18:16:33,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/a6fcd18af1c24409a09f56076e5dbf70 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6fcd18af1c24409a09f56076e5dbf70 2024-11-10T18:16:33,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6fcd18af1c24409a09f56076e5dbf70, entries=150, sequenceid=318, filesize=12.0 K 2024-11-10T18:16:33,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 01f94435f1c5387043f08668493d3e9e in 1663ms, sequenceid=318, compaction requested=true 2024-11-10T18:16:33,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:A, priority=-2147483648, current under compaction store size is 1 2024-11-10T18:16:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:33,650 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:B, priority=-2147483648, current under compaction store size is 2 2024-11-10T18:16:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 01f94435f1c5387043f08668493d3e9e:C, priority=-2147483648, current under compaction store size is 3 2024-11-10T18:16:33,650 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:33,650 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:33,650 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:33,650 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:33,650 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/B is initiating minor compaction (all files) 2024-11-10T18:16:33,650 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/A is initiating minor compaction (all files) 2024-11-10T18:16:33,650 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/B in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:33,650 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/A in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:33,651 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/94b5a9c62a174edf9e190021c1364272, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/c84fbfadc31c4e62a60ebcb29782092a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/8d33d209163d4660bd3ddac2b3e53276] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=36.7 K 2024-11-10T18:16:33,651 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d064571f9be74d1c8b3da48ac2a1996c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=92.2 K 2024-11-10T18:16:33,651 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. files: [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d064571f9be74d1c8b3da48ac2a1996c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6] 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 94b5a9c62a174edf9e190021c1364272, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262587933 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting d064571f9be74d1c8b3da48ac2a1996c, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262587933 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting c84fbfadc31c4e62a60ebcb29782092a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731262589115 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34e747766e2147339ad8b4c67b98de2a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731262589115 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d33d209163d4660bd3ddac2b3e53276, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731262591352 2024-11-10T18:16:33,651 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7dc3f8b9b36d42c7a5652a9c88deddf6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731262591352 2024-11-10T18:16:33,656 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:33,658 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241110c5642cc14ce1481b92d5f349fedf2f92_01f94435f1c5387043f08668493d3e9e store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:33,658 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#B#compaction#502 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:33,659 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/ebf8cffa86734d929a794a5685f2611c is 50, key is test_row_0/B:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:33,660 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241110c5642cc14ce1481b92d5f349fedf2f92_01f94435f1c5387043f08668493d3e9e, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:33,660 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241110c5642cc14ce1481b92d5f349fedf2f92_01f94435f1c5387043f08668493d3e9e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:33,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742422_1598 (size=13051) 2024-11-10T18:16:33,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742423_1599 (size=4469) 2024-11-10T18:16:34,064 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#A#compaction#501 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:34,064 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/c817ff7dfb0a413c96a8a4b55482fe06 is 175, key is test_row_0/A:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:34,066 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/ebf8cffa86734d929a794a5685f2611c as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ebf8cffa86734d929a794a5685f2611c 2024-11-10T18:16:34,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742424_1600 (size=32005) 2024-11-10T18:16:34,070 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/B of 01f94435f1c5387043f08668493d3e9e into ebf8cffa86734d929a794a5685f2611c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:34,070 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:34,070 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/B, priority=13, startTime=1731262593650; duration=0sec 2024-11-10T18:16:34,070 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-10T18:16:34,070 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:B 2024-11-10T18:16:34,070 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-10T18:16:34,070 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-10T18:16:34,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1541): 01f94435f1c5387043f08668493d3e9e/C is initiating minor compaction (all files) 2024-11-10T18:16:34,071 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 01f94435f1c5387043f08668493d3e9e/C in TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:34,071 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/da264a9e2e4f4e31860bdd831c24d3d9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/81446cb8f54c461aaa344b8e74a4dd04, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6fcd18af1c24409a09f56076e5dbf70] into tmpdir=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp, totalSize=36.7 K 2024-11-10T18:16:34,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting da264a9e2e4f4e31860bdd831c24d3d9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731262587933 2024-11-10T18:16:34,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting 81446cb8f54c461aaa344b8e74a4dd04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731262589115 2024-11-10T18:16:34,071 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] compactions.Compactor(225): Compacting a6fcd18af1c24409a09f56076e5dbf70, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731262591352 2024-11-10T18:16:34,076 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 01f94435f1c5387043f08668493d3e9e#C#compaction#503 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-10T18:16:34,076 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/c25bc0187fed423287049065dcdabedf is 50, key is test_row_0/C:col10/1731262591352/Put/seqid=0 2024-11-10T18:16:34,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742425_1601 (size=13051) 2024-11-10T18:16:34,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=32909 {}] regionserver.HRegion(8855): Flush requested on 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:34,206 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-10T18:16:34,207 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-11-10T18:16:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:34,207 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:34,207 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:34,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:34,207 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:34,207 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:34,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d200ff0bdf54793b70409064ecb6d82_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262592046/Put/seqid=0 2024-11-10T18:16:34,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742426_1602 (size=12454) 2024-11-10T18:16:34,471 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/c817ff7dfb0a413c96a8a4b55482fe06 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/c817ff7dfb0a413c96a8a4b55482fe06 2024-11-10T18:16:34,475 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/A of 01f94435f1c5387043f08668493d3e9e into c817ff7dfb0a413c96a8a4b55482fe06(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:34,475 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:34,475 INFO [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/A, priority=13, startTime=1731262593649; duration=0sec 2024-11-10T18:16:34,475 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:34,475 DEBUG [RS:0;691ea56905d2:32909-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:A 2024-11-10T18:16:34,482 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/c25bc0187fed423287049065dcdabedf as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/c25bc0187fed423287049065dcdabedf 2024-11-10T18:16:34,485 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 01f94435f1c5387043f08668493d3e9e/C of 01f94435f1c5387043f08668493d3e9e into c25bc0187fed423287049065dcdabedf(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-10T18:16:34,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:34,485 INFO [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., storeName=01f94435f1c5387043f08668493d3e9e/C, priority=13, startTime=1731262593650; duration=0sec 2024-11-10T18:16:34,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-10T18:16:34,485 DEBUG [RS:0;691ea56905d2:32909-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 01f94435f1c5387043f08668493d3e9e:C 2024-11-10T18:16:34,615 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:34,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102d200ff0bdf54793b70409064ecb6d82_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d200ff0bdf54793b70409064ecb6d82_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:34,618 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33e9714721504a6a95bee30c9e984cb1, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:34,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33e9714721504a6a95bee30c9e984cb1 is 175, key is test_row_0/A:col10/1731262592046/Put/seqid=0 2024-11-10T18:16:34,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742427_1603 (size=31255) 2024-11-10T18:16:35,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33e9714721504a6a95bee30c9e984cb1 2024-11-10T18:16:35,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/1ed87623caad40d68e164aee48437891 is 50, key is test_row_0/B:col10/1731262592046/Put/seqid=0 2024-11-10T18:16:35,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742428_1604 (size=12301) 2024-11-10T18:16:35,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/1ed87623caad40d68e164aee48437891 2024-11-10T18:16:35,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/e515828abf704decbad456ab9317c38f is 50, key is test_row_0/C:col10/1731262592046/Put/seqid=0 2024-11-10T18:16:35,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742429_1605 (size=12301) 2024-11-10T18:16:35,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/e515828abf704decbad456ab9317c38f 2024-11-10T18:16:35,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/33e9714721504a6a95bee30c9e984cb1 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33e9714721504a6a95bee30c9e984cb1 2024-11-10T18:16:35,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33e9714721504a6a95bee30c9e984cb1, entries=150, sequenceid=337, filesize=30.5 K 2024-11-10T18:16:35,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/1ed87623caad40d68e164aee48437891 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/1ed87623caad40d68e164aee48437891 2024-11-10T18:16:35,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/1ed87623caad40d68e164aee48437891, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T18:16:35,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/e515828abf704decbad456ab9317c38f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/e515828abf704decbad456ab9317c38f 2024-11-10T18:16:35,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/e515828abf704decbad456ab9317c38f, entries=150, sequenceid=337, filesize=12.0 K 2024-11-10T18:16:35,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 01f94435f1c5387043f08668493d3e9e in 1645ms, sequenceid=337, compaction requested=false 2024-11-10T18:16:35,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 01f94435f1c5387043f08668493d3e9e: 2024-11-10T18:16:36,556 INFO [Thread-2299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2299. 2024-11-10T18:16:36,556 DEBUG [Thread-2299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:36,556 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:36,556 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:36,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:43,466 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-11-10T18:16:43,466 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:43,466 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,466 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:43,476 INFO [Thread-2301 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2301. 2024-11-10T18:16:43,476 DEBUG [Thread-2301 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:43,476 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,476 DEBUG [Thread-2301 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:43,556 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-11-10T18:16:43,556 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-10T18:16:43,556 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,556 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,556 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2711 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2701 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2689 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2705 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2715 2024-11-10T18:16:43,556 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-10T18:16:43,557 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:16:43,557 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:16:43,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:43,557 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:43,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-10T18:16:43,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:43,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T18:16:43,560 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262603560"}]},"ts":"1731262603560"} 2024-11-10T18:16:43,561 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-10T18:16:43,561 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-10T18:16:43,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-10T18:16:43,562 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, UNASSIGN}] 2024-11-10T18:16:43,563 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, UNASSIGN 2024-11-10T18:16:43,563 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=CLOSING, regionLocation=691ea56905d2,32909,1731262402364 2024-11-10T18:16:43,564 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, UNASSIGN because future has completed 2024-11-10T18:16:43,564 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-10T18:16:43,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364}] 2024-11-10T18:16:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T18:16:43,717 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(122): Close 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1722): Closing 01f94435f1c5387043f08668493d3e9e, disabling compactions & flushes 2024-11-10T18:16:43,717 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. after waiting 0 ms 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:43,717 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(2902): Flushing 01f94435f1c5387043f08668493d3e9e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=A 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=B 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 01f94435f1c5387043f08668493d3e9e, store=C 2024-11-10T18:16:43,717 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-10T18:16:43,722 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102fdc7210b70049be8179e6d3d3ebfde6_01f94435f1c5387043f08668493d3e9e is 50, key is test_row_0/A:col10/1731262596555/Put/seqid=0 2024-11-10T18:16:43,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742430_1606 (size=12454) 2024-11-10T18:16:43,725 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T18:16:43,728 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411102fdc7210b70049be8179e6d3d3ebfde6_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102fdc7210b70049be8179e6d3d3ebfde6_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:43,728 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/8cad1a9a809242e4bc5b0ea95a642793, store: [table=TestAcidGuarantees family=A region=01f94435f1c5387043f08668493d3e9e] 2024-11-10T18:16:43,729 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/8cad1a9a809242e4bc5b0ea95a642793 is 175, key is test_row_0/A:col10/1731262596555/Put/seqid=0 2024-11-10T18:16:43,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742431_1607 (size=31255) 2024-11-10T18:16:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T18:16:44,132 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=346, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/8cad1a9a809242e4bc5b0ea95a642793 2024-11-10T18:16:44,137 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/dd17c77a8d4d4c95b5906cf30be1e6ca is 50, key is test_row_0/B:col10/1731262596555/Put/seqid=0 2024-11-10T18:16:44,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742432_1608 (size=12301) 2024-11-10T18:16:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T18:16:44,540 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/dd17c77a8d4d4c95b5906cf30be1e6ca 2024-11-10T18:16:44,545 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/973de893fd534bdaadd8ea9b29caef8f is 50, key is test_row_0/C:col10/1731262596555/Put/seqid=0 2024-11-10T18:16:44,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742433_1609 (size=12301) 2024-11-10T18:16:44,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T18:16:44,949 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/973de893fd534bdaadd8ea9b29caef8f 2024-11-10T18:16:44,952 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/A/8cad1a9a809242e4bc5b0ea95a642793 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/8cad1a9a809242e4bc5b0ea95a642793 2024-11-10T18:16:44,955 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/8cad1a9a809242e4bc5b0ea95a642793, entries=150, sequenceid=346, filesize=30.5 K 2024-11-10T18:16:44,955 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/B/dd17c77a8d4d4c95b5906cf30be1e6ca as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/dd17c77a8d4d4c95b5906cf30be1e6ca 2024-11-10T18:16:44,957 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/dd17c77a8d4d4c95b5906cf30be1e6ca, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T18:16:44,958 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/.tmp/C/973de893fd534bdaadd8ea9b29caef8f as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/973de893fd534bdaadd8ea9b29caef8f 2024-11-10T18:16:44,960 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/973de893fd534bdaadd8ea9b29caef8f, entries=150, sequenceid=346, filesize=12.0 K 2024-11-10T18:16:44,961 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 01f94435f1c5387043f08668493d3e9e in 1244ms, sequenceid=346, compaction requested=true 2024-11-10T18:16:44,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/bb1e1d546af241078520ea69d3a049fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/f234d3f2d5854b78bba984854cb60b33, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0ce98ebe66d148288a55fd09dbcc021f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/78e3289c41374176923835c38e46a828, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/fc0d51f6b0a5416e96388826da8ad545, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d064571f9be74d1c8b3da48ac2a1996c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6] to archive 2024-11-10T18:16:44,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:16:44,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7bfe5dc3c7db42e3a2bb4da609538d57 2024-11-10T18:16:44,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/af3449ab395646d7a0adffa4d8a02b04 2024-11-10T18:16:44,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/bb1e1d546af241078520ea69d3a049fb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/bb1e1d546af241078520ea69d3a049fb 2024-11-10T18:16:44,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/51bee30119044fed933ac10f50802746 2024-11-10T18:16:44,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/e97c24dcb49342eb8a051f2f9b3e4d7c 2024-11-10T18:16:44,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33631d85a1464f3a8340d9c5674977ea 2024-11-10T18:16:44,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/f234d3f2d5854b78bba984854cb60b33 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/f234d3f2d5854b78bba984854cb60b33 2024-11-10T18:16:44,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/aeb7e3aad68d493387ed8b4e5f09c1b9 2024-11-10T18:16:44,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5201bb957741454bba815900515fb8cd 2024-11-10T18:16:44,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5a29107d3bfc402fbcbe8213d2f88688 2024-11-10T18:16:44,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0ce98ebe66d148288a55fd09dbcc021f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0ce98ebe66d148288a55fd09dbcc021f 2024-11-10T18:16:44,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d23370e1f3fb4cbd8eea111d6a08f9e8 2024-11-10T18:16:44,972 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/ed7c44c4b880471b91cd644da5767359 2024-11-10T18:16:44,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/78e3289c41374176923835c38e46a828 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/78e3289c41374176923835c38e46a828 2024-11-10T18:16:44,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7de840af1ab64600b28f70a8261da945 2024-11-10T18:16:44,974 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/fc0d51f6b0a5416e96388826da8ad545 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/fc0d51f6b0a5416e96388826da8ad545 2024-11-10T18:16:44,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/5ceecf3d5175493291cb93f1b1f4ca9e 2024-11-10T18:16:44,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/387149e12b9d48fab02177d69b64838e 2024-11-10T18:16:44,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/0aee7a76539e487093c700ebdb27e8cf 2024-11-10T18:16:44,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d064571f9be74d1c8b3da48ac2a1996c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/d064571f9be74d1c8b3da48ac2a1996c 2024-11-10T18:16:44,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/34e747766e2147339ad8b4c67b98de2a 2024-11-10T18:16:44,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/7dc3f8b9b36d42c7a5652a9c88deddf6 2024-11-10T18:16:44,980 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/42a3972e9431402386f85237aebc6aa9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/169b829c9c024a38a50248d3048ef40b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7a811001447a4e608992cbd4c53beb16, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/17119f4305814bfaab00dda27081bade, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/4755806e9ef64f60b31c896702313820, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/16ab32ae684148ff8b8f4133e44b06fb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/65fa20f379b446fdb16ef61fa77a7fa7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/59244f0ac39143edbb7451b6677ae739, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/20be7750453f41d9b2782be3c6130903, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/2ca3f08e05b54ad4b9a350ff1a654a56, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ee3811b1041b4755b466ba95140dac6b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/0ba05f7a80e5430e9300462936ed28e4, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d6dc255340914952aacea24fd2477341, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d52f3a470dbd4416a1ffaa92b4574093, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/40b2cfd22e5c4d5bb8766b1965a86360, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/136a035799a5404aa7ad13e82c45231b, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/55f9644e5269456c9780adf6f60876c7, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/3531d866e1594513acfd68d105852521, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/94b5a9c62a174edf9e190021c1364272, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7cb0b122f14f43ecbb9ac51ceb0744f2, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/c84fbfadc31c4e62a60ebcb29782092a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/8d33d209163d4660bd3ddac2b3e53276] to archive 2024-11-10T18:16:44,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:16:44,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/42a3972e9431402386f85237aebc6aa9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/42a3972e9431402386f85237aebc6aa9 2024-11-10T18:16:44,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/169b829c9c024a38a50248d3048ef40b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/169b829c9c024a38a50248d3048ef40b 2024-11-10T18:16:44,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7a811001447a4e608992cbd4c53beb16 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7a811001447a4e608992cbd4c53beb16 2024-11-10T18:16:44,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/17119f4305814bfaab00dda27081bade to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/17119f4305814bfaab00dda27081bade 2024-11-10T18:16:44,985 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/4755806e9ef64f60b31c896702313820 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/4755806e9ef64f60b31c896702313820 2024-11-10T18:16:44,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/16ab32ae684148ff8b8f4133e44b06fb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/16ab32ae684148ff8b8f4133e44b06fb 2024-11-10T18:16:44,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/65fa20f379b446fdb16ef61fa77a7fa7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/65fa20f379b446fdb16ef61fa77a7fa7 2024-11-10T18:16:44,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/59244f0ac39143edbb7451b6677ae739 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/59244f0ac39143edbb7451b6677ae739 2024-11-10T18:16:44,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/20be7750453f41d9b2782be3c6130903 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/20be7750453f41d9b2782be3c6130903 2024-11-10T18:16:44,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/2ca3f08e05b54ad4b9a350ff1a654a56 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/2ca3f08e05b54ad4b9a350ff1a654a56 2024-11-10T18:16:44,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ee3811b1041b4755b466ba95140dac6b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ee3811b1041b4755b466ba95140dac6b 2024-11-10T18:16:44,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/0ba05f7a80e5430e9300462936ed28e4 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/0ba05f7a80e5430e9300462936ed28e4 2024-11-10T18:16:44,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d6dc255340914952aacea24fd2477341 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d6dc255340914952aacea24fd2477341 2024-11-10T18:16:44,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d52f3a470dbd4416a1ffaa92b4574093 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/d52f3a470dbd4416a1ffaa92b4574093 2024-11-10T18:16:44,993 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/40b2cfd22e5c4d5bb8766b1965a86360 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/40b2cfd22e5c4d5bb8766b1965a86360 2024-11-10T18:16:44,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/136a035799a5404aa7ad13e82c45231b to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/136a035799a5404aa7ad13e82c45231b 2024-11-10T18:16:44,995 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/55f9644e5269456c9780adf6f60876c7 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/55f9644e5269456c9780adf6f60876c7 2024-11-10T18:16:44,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/3531d866e1594513acfd68d105852521 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/3531d866e1594513acfd68d105852521 2024-11-10T18:16:44,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/94b5a9c62a174edf9e190021c1364272 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/94b5a9c62a174edf9e190021c1364272 2024-11-10T18:16:44,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7cb0b122f14f43ecbb9ac51ceb0744f2 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/7cb0b122f14f43ecbb9ac51ceb0744f2 2024-11-10T18:16:44,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/c84fbfadc31c4e62a60ebcb29782092a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/c84fbfadc31c4e62a60ebcb29782092a 2024-11-10T18:16:44,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/8d33d209163d4660bd3ddac2b3e53276 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/8d33d209163d4660bd3ddac2b3e53276 2024-11-10T18:16:45,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/b53f0b1f83554825bdc01f61a0584deb, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/97885dc128984010912da1cc327fb2b6, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/bae1c7ad11ba4036a8be09b68b21945a, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/6839dfc42d454e72962fa9b4372717e3, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/f75a0cc4d0a94fd199a4986b9aeedd48, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/dfa21e1cf9b04a56b2dcf403407ed39f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/340563790411484e84bcbe790d296c18, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5b9612c7508349dda360ee1ecee1a034, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/67efaeb744fc46ef86414d2df17d6869, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe12d3df72364d6b93578131ce352b5c, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6111ca3117446ca918678824ad7b65d, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5215df632fe341078c86d6a153bdccef, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9068fc0e4e064c33bc78211c34b4fbb8, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/d1b25c0d007d48508500f0c62e584c52, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9e0766e48f944969945b217c23463a6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe52041968de428dae71f6250c17069f, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a5897bbf299d4000b35709201ef57aee, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/57a661d374364a81a8fe11f0087dfa6e, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/da264a9e2e4f4e31860bdd831c24d3d9, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/cf1db2ae6e7f430491654dafb1685b68, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/81446cb8f54c461aaa344b8e74a4dd04, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6fcd18af1c24409a09f56076e5dbf70] to archive 2024-11-10T18:16:45,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-10T18:16:45,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/b53f0b1f83554825bdc01f61a0584deb to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/b53f0b1f83554825bdc01f61a0584deb 2024-11-10T18:16:45,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/97885dc128984010912da1cc327fb2b6 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/97885dc128984010912da1cc327fb2b6 2024-11-10T18:16:45,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/bae1c7ad11ba4036a8be09b68b21945a to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/bae1c7ad11ba4036a8be09b68b21945a 2024-11-10T18:16:45,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/6839dfc42d454e72962fa9b4372717e3 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/6839dfc42d454e72962fa9b4372717e3 2024-11-10T18:16:45,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/f75a0cc4d0a94fd199a4986b9aeedd48 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/f75a0cc4d0a94fd199a4986b9aeedd48 2024-11-10T18:16:45,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/dfa21e1cf9b04a56b2dcf403407ed39f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/dfa21e1cf9b04a56b2dcf403407ed39f 2024-11-10T18:16:45,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/340563790411484e84bcbe790d296c18 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/340563790411484e84bcbe790d296c18 2024-11-10T18:16:45,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5b9612c7508349dda360ee1ecee1a034 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5b9612c7508349dda360ee1ecee1a034 2024-11-10T18:16:45,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/67efaeb744fc46ef86414d2df17d6869 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/67efaeb744fc46ef86414d2df17d6869 2024-11-10T18:16:45,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe12d3df72364d6b93578131ce352b5c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe12d3df72364d6b93578131ce352b5c 2024-11-10T18:16:45,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6111ca3117446ca918678824ad7b65d to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6111ca3117446ca918678824ad7b65d 2024-11-10T18:16:45,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5215df632fe341078c86d6a153bdccef to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/5215df632fe341078c86d6a153bdccef 2024-11-10T18:16:45,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9068fc0e4e064c33bc78211c34b4fbb8 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9068fc0e4e064c33bc78211c34b4fbb8 2024-11-10T18:16:45,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/d1b25c0d007d48508500f0c62e584c52 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/d1b25c0d007d48508500f0c62e584c52 2024-11-10T18:16:45,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9e0766e48f944969945b217c23463a6e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/9e0766e48f944969945b217c23463a6e 2024-11-10T18:16:45,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe52041968de428dae71f6250c17069f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/fe52041968de428dae71f6250c17069f 2024-11-10T18:16:45,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a5897bbf299d4000b35709201ef57aee to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a5897bbf299d4000b35709201ef57aee 2024-11-10T18:16:45,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/57a661d374364a81a8fe11f0087dfa6e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/57a661d374364a81a8fe11f0087dfa6e 2024-11-10T18:16:45,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/da264a9e2e4f4e31860bdd831c24d3d9 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/da264a9e2e4f4e31860bdd831c24d3d9 2024-11-10T18:16:45,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/cf1db2ae6e7f430491654dafb1685b68 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/cf1db2ae6e7f430491654dafb1685b68 2024-11-10T18:16:45,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/81446cb8f54c461aaa344b8e74a4dd04 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/81446cb8f54c461aaa344b8e74a4dd04 2024-11-10T18:16:45,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6fcd18af1c24409a09f56076e5dbf70 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/a6fcd18af1c24409a09f56076e5dbf70 2024-11-10T18:16:45,021 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/recovered.edits/349.seqid, newMaxSeqId=349, maxSeqId=4 2024-11-10T18:16:45,021 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. 2024-11-10T18:16:45,022 DEBUG [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] regionserver.HRegion(1676): Region close journal for 01f94435f1c5387043f08668493d3e9e: Waiting for close lock at 1731262603717Running coprocessor pre-close hooks at 1731262603717Disabling compacts and flushes for region at 1731262603717Disabling writes for close at 1731262603717Obtaining lock to block concurrent updates at 1731262603717Preparing flush snapshotting stores in 01f94435f1c5387043f08668493d3e9e at 1731262603717Finished memstore snapshotting TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731262603717Flushing stores of TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e. at 1731262603718 (+1 ms)Flushing 01f94435f1c5387043f08668493d3e9e/A: creating writer at 1731262603718Flushing mob file 01f94435f1c5387043f08668493d3e9e/A: appending metadata at 1731262603722 (+4 ms)Flushing mob file 01f94435f1c5387043f08668493d3e9e/A: closing flushed file at 1731262603722Flushing 01f94435f1c5387043f08668493d3e9e/A: appending metadata at 1731262603728 (+6 ms)Flushing 01f94435f1c5387043f08668493d3e9e/A: closing flushed file at 1731262603728Flushing 01f94435f1c5387043f08668493d3e9e/B: creating writer at 1731262604135 (+407 ms)Flushing 01f94435f1c5387043f08668493d3e9e/B: appending metadata at 1731262604137 (+2 ms)Flushing 01f94435f1c5387043f08668493d3e9e/B: closing flushed file at 1731262604137Flushing 01f94435f1c5387043f08668493d3e9e/C: creating writer at 1731262604543 (+406 ms)Flushing 01f94435f1c5387043f08668493d3e9e/C: appending metadata at 1731262604545 (+2 ms)Flushing 01f94435f1c5387043f08668493d3e9e/C: closing flushed file at 1731262604545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d445f12: reopening flushed file at 1731262604951 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1353cf04: reopening flushed file at 1731262604955 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29bf7bd3: reopening flushed file at 1731262604957 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 01f94435f1c5387043f08668493d3e9e in 1244ms, sequenceid=346, compaction requested=true at 1731262604961 (+4 ms)Writing region close event to WAL at 1731262605019 (+58 ms)Running coprocessor post-close hooks at 1731262605021 (+2 ms)Closed at 1731262605021 2024-11-10T18:16:45,023 INFO [RS_CLOSE_REGION-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=159}] handler.UnassignRegionHandler(157): Closed 01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,023 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=158 updating hbase:meta row=01f94435f1c5387043f08668493d3e9e, regionState=CLOSED 2024-11-10T18:16:45,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE, hasLock=false; CloseRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 because future has completed 2024-11-10T18:16:45,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-10T18:16:45,026 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseRegionProcedure 01f94435f1c5387043f08668493d3e9e, server=691ea56905d2,32909,1731262402364 in 1.4600 sec 2024-11-10T18:16:45,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-10T18:16:45,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=01f94435f1c5387043f08668493d3e9e, UNASSIGN in 1.4640 sec 2024-11-10T18:16:45,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-10T18:16:45,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4670 sec 2024-11-10T18:16:45,030 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731262605030"}]},"ts":"1731262605030"} 2024-11-10T18:16:45,031 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-10T18:16:45,031 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-10T18:16:45,033 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-10T18:16:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-10T18:16:45,705 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:45,706 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-10T18:16:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:45,707 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=160, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T18:16:45,708 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=160, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:45,710 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,712 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C, FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/recovered.edits] 2024-11-10T18:16:45,713 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33e9714721504a6a95bee30c9e984cb1 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/33e9714721504a6a95bee30c9e984cb1 2024-11-10T18:16:45,714 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/8cad1a9a809242e4bc5b0ea95a642793 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/8cad1a9a809242e4bc5b0ea95a642793 2024-11-10T18:16:45,715 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/c817ff7dfb0a413c96a8a4b55482fe06 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/A/c817ff7dfb0a413c96a8a4b55482fe06 2024-11-10T18:16:45,717 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/1ed87623caad40d68e164aee48437891 to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/1ed87623caad40d68e164aee48437891 2024-11-10T18:16:45,717 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/dd17c77a8d4d4c95b5906cf30be1e6ca to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/dd17c77a8d4d4c95b5906cf30be1e6ca 2024-11-10T18:16:45,718 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ebf8cffa86734d929a794a5685f2611c to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/B/ebf8cffa86734d929a794a5685f2611c 2024-11-10T18:16:45,720 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/973de893fd534bdaadd8ea9b29caef8f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/973de893fd534bdaadd8ea9b29caef8f 2024-11-10T18:16:45,720 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/c25bc0187fed423287049065dcdabedf to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/c25bc0187fed423287049065dcdabedf 2024-11-10T18:16:45,721 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/e515828abf704decbad456ab9317c38f to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/C/e515828abf704decbad456ab9317c38f 2024-11-10T18:16:45,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/recovered.edits/349.seqid to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e/recovered.edits/349.seqid 2024-11-10T18:16:45,723 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/default/TestAcidGuarantees/01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-10T18:16:45,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T18:16:45,724 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-10T18:16:45,726 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100a05fcbba9b64015a3f5d631dc07def6_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411100a05fcbba9b64015a3f5d631dc07def6_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,727 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d200ff0bdf54793b70409064ecb6d82_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102d200ff0bdf54793b70409064ecb6d82_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,727 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e1ba973997c4d62b466b7d46f70f418_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102e1ba973997c4d62b466b7d46f70f418_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,728 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102fdc7210b70049be8179e6d3d3ebfde6_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411102fdc7210b70049be8179e6d3d3ebfde6_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,729 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111047623871f0d74e7690e8c7be2178d325_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111047623871f0d74e7690e8c7be2178d325_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,730 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c2fc57294e5483d9f2661d376be224a_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411104c2fc57294e5483d9f2661d376be224a_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106037065274da41cdaea0bd88a485e167_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411106037065274da41cdaea0bd88a485e167_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,731 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111070590cfb00be443c90e0cbf935ecf969_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111070590cfb00be443c90e0cbf935ecf969_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,732 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111086ba5f40e2d6428b945aa22ef34b4b81_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111086ba5f40e2d6428b945aa22ef34b4b81_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,733 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a54a66c706ee4449af9f7826c5762d6c_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110a54a66c706ee4449af9f7826c5762d6c_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba1f15dfe2e047f79c1bc904f0900624_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ba1f15dfe2e047f79c1bc904f0900624_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,734 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d25de3fbda1e46f9a73ed187e3e661fa_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d25de3fbda1e46f9a73ed187e3e661fa_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,735 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2b1e34c65dd4ae19b37fa00cb6c6c50_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2b1e34c65dd4ae19b37fa00cb6c6c50_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,736 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2cb6b3d1d3049f08da5ca4649afc615_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d2cb6b3d1d3049f08da5ca4649afc615_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,737 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d6ec5d2d4d434f03a642a4067930ccdd_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d6ec5d2d4d434f03a642a4067930ccdd_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,738 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9c312b05a314b65bc780e21a8b72d0a_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110d9c312b05a314b65bc780e21a8b72d0a_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,738 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcb6bc304d794a16821386420e9022b8_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110dcb6bc304d794a16821386420e9022b8_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,739 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ead927c83fe54251a55fd9ba0a74e5ff_01f94435f1c5387043f08668493d3e9e to hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241110ead927c83fe54251a55fd9ba0a74e5ff_01f94435f1c5387043f08668493d3e9e 2024-11-10T18:16:45,739 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-10T18:16:45,742 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=160, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:45,743 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-10T18:16:45,745 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-10T18:16:45,746 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=160, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:45,746 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-10T18:16:45,746 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731262605746"}]},"ts":"9223372036854775807"} 2024-11-10T18:16:45,748 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T18:16:45,748 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 01f94435f1c5387043f08668493d3e9e, NAME => 'TestAcidGuarantees,,1731262570442.01f94435f1c5387043f08668493d3e9e.', STARTKEY => '', ENDKEY => ''}] 2024-11-10T18:16:45,748 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-10T18:16:45,748 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731262605748"}]},"ts":"9223372036854775807"} 2024-11-10T18:16:45,750 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-10T18:16:45,750 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=160, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-10T18:16:45,751 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-10T18:16:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=41343 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-10T18:16:45,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-10T18:16:45,815 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-10T18:16:45,826 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 247), OpenFileDescriptor=443 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=224 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=7963 (was 7996) 2024-11-10T18:16:45,826 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T18:16:45,826 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:16:45,826 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:16:45,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:45,826 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:45,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T18:16:45,827 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T18:16:45,827 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=974344401, stopped=false 2024-11-10T18:16:45,827 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=691ea56905d2,41343,1731262401541 2024-11-10T18:16:45,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T18:16:45,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T18:16:45,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:16:45,829 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:16:45,829 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T18:16:45,830 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T18:16:45,830 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:16:45,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:45,830 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T18:16:45,830 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '691ea56905d2,32909,1731262402364' ***** 2024-11-10T18:16:45,830 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T18:16:45,830 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T18:16:45,830 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T18:16:45,831 INFO [RS:0;691ea56905d2:32909 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T18:16:45,831 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T18:16:45,831 INFO [RS:0;691ea56905d2:32909 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T18:16:45,831 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(959): stopping server 691ea56905d2,32909,1731262402364 2024-11-10T18:16:45,831 INFO [RS:0;691ea56905d2:32909 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T18:16:45,831 INFO [RS:0;691ea56905d2:32909 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;691ea56905d2:32909. 2024-11-10T18:16:45,831 DEBUG [RS:0;691ea56905d2:32909 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T18:16:45,831 DEBUG [RS:0;691ea56905d2:32909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:45,832 INFO [RS:0;691ea56905d2:32909 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T18:16:45,832 INFO [RS:0;691ea56905d2:32909 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T18:16:45,832 INFO [RS:0;691ea56905d2:32909 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T18:16:45,832 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T18:16:45,833 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T18:16:45,833 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-10T18:16:45,833 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T18:16:45,833 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:45,833 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T18:16:45,833 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T18:16:45,833 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T18:16:45,833 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T18:16:45,833 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-10T18:16:45,857 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/info/9f4383ff10fd476cb00f10237d1f0ab6 is 95, key is TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94./info:/1731262439976/DeleteFamily/seqid=0 2024-11-10T18:16:45,860 INFO [regionserver/691ea56905d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T18:16:45,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742434_1610 (size=6514) 2024-11-10T18:16:45,883 INFO [regionserver/691ea56905d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-10T18:16:45,883 INFO [regionserver/691ea56905d2:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-10T18:16:46,033 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:46,233 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:46,260 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/info/9f4383ff10fd476cb00f10237d1f0ab6 2024-11-10T18:16:46,278 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/ns/bc1baa77d36b4104bada24f1b1cc3cf4 is 93, key is TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94./ns:/1731262439929/DeleteFamily/seqid=0 2024-11-10T18:16:46,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742435_1611 (size=5906) 2024-11-10T18:16:46,433 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:46,634 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:46,682 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/ns/bc1baa77d36b4104bada24f1b1cc3cf4 2024-11-10T18:16:46,699 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/rep_barrier/699ef66791a74563ad12c4e29348d7be is 102, key is TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94./rep_barrier:/1731262439929/DeleteFamily/seqid=0 2024-11-10T18:16:46,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742436_1612 (size=6025) 2024-11-10T18:16:46,834 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:47,034 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:47,102 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/rep_barrier/699ef66791a74563ad12c4e29348d7be 2024-11-10T18:16:47,119 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/table/ecaa527cb4e74fe4bb8807950a582792 is 96, key is TestAcidGuarantees,,1731262406420.17351107c8cbd49315660faf09692c94./table:/1731262439929/DeleteFamily/seqid=0 2024-11-10T18:16:47,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742437_1613 (size=5936) 2024-11-10T18:16:47,234 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:47,434 DEBUG [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-10T18:16:47,522 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/table/ecaa527cb4e74fe4bb8807950a582792 2024-11-10T18:16:47,526 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/info/9f4383ff10fd476cb00f10237d1f0ab6 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/info/9f4383ff10fd476cb00f10237d1f0ab6 2024-11-10T18:16:47,528 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/info/9f4383ff10fd476cb00f10237d1f0ab6, entries=12, sequenceid=90, filesize=6.4 K 2024-11-10T18:16:47,529 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/ns/bc1baa77d36b4104bada24f1b1cc3cf4 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/ns/bc1baa77d36b4104bada24f1b1cc3cf4 2024-11-10T18:16:47,531 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/ns/bc1baa77d36b4104bada24f1b1cc3cf4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-10T18:16:47,531 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/rep_barrier/699ef66791a74563ad12c4e29348d7be as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/rep_barrier/699ef66791a74563ad12c4e29348d7be 2024-11-10T18:16:47,534 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/rep_barrier/699ef66791a74563ad12c4e29348d7be, entries=6, sequenceid=90, filesize=5.9 K 2024-11-10T18:16:47,534 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/.tmp/table/ecaa527cb4e74fe4bb8807950a582792 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/table/ecaa527cb4e74fe4bb8807950a582792 2024-11-10T18:16:47,536 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/table/ecaa527cb4e74fe4bb8807950a582792, entries=7, sequenceid=90, filesize=5.8 K 2024-11-10T18:16:47,537 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false 2024-11-10T18:16:47,541 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-10T18:16:47,541 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T18:16:47,541 INFO [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T18:16:47,541 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731262605833Running coprocessor pre-close hooks at 1731262605833Disabling compacts and flushes for region at 1731262605833Disabling writes for close at 1731262605833Obtaining lock to block concurrent updates at 1731262605833Preparing flush snapshotting stores in 1588230740 at 1731262605833Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731262605834 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731262605834Flushing 1588230740/info: creating writer at 1731262605834Flushing 1588230740/info: appending metadata at 1731262605856 (+22 ms)Flushing 1588230740/info: closing flushed file at 1731262605856Flushing 1588230740/ns: creating writer at 1731262606264 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731262606278 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731262606278Flushing 1588230740/rep_barrier: creating writer at 1731262606685 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731262606698 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731262606698Flushing 1588230740/table: creating writer at 1731262607105 (+407 ms)Flushing 1588230740/table: appending metadata at 1731262607119 (+14 ms)Flushing 1588230740/table: closing flushed file at 1731262607119Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b437047: reopening flushed file at 1731262607525 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@798e55d0: reopening flushed file at 1731262607528 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@766401d9: reopening flushed file at 1731262607531 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ff9aa6d: reopening flushed file at 1731262607534 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1704ms, sequenceid=90, compaction requested=false at 1731262607537 (+3 ms)Writing region close event to WAL at 1731262607538 (+1 ms)Running coprocessor post-close hooks at 1731262607541 (+3 ms)Closed at 1731262607541 2024-11-10T18:16:47,541 DEBUG [RS_CLOSE_META-regionserver/691ea56905d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T18:16:47,635 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(976): stopping server 691ea56905d2,32909,1731262402364; all regions closed. 2024-11-10T18:16:47,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741834_1010 (size=25344) 2024-11-10T18:16:47,641 DEBUG [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/oldWALs 2024-11-10T18:16:47,641 INFO [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 691ea56905d2%2C32909%2C1731262402364.meta:.meta(num 1731262406001) 2024-11-10T18:16:47,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741832_1008 (size=15982755) 2024-11-10T18:16:47,644 DEBUG [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/oldWALs 2024-11-10T18:16:47,644 INFO [RS:0;691ea56905d2:32909 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 691ea56905d2%2C32909%2C1731262402364:(num 1731262405043) 2024-11-10T18:16:47,644 DEBUG [RS:0;691ea56905d2:32909 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T18:16:47,644 INFO [RS:0;691ea56905d2:32909 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T18:16:47,644 INFO [RS:0;691ea56905d2:32909 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T18:16:47,644 INFO [RS:0;691ea56905d2:32909 {}] hbase.ChoreService(370): Chore service for: regionserver/691ea56905d2:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T18:16:47,644 INFO [RS:0;691ea56905d2:32909 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T18:16:47,644 INFO [regionserver/691ea56905d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T18:16:47,646 INFO [RS:0;691ea56905d2:32909 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32909 2024-11-10T18:16:47,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/691ea56905d2,32909,1731262402364 2024-11-10T18:16:47,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T18:16:47,650 INFO [RS:0;691ea56905d2:32909 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T18:16:47,652 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [691ea56905d2,32909,1731262402364] 2024-11-10T18:16:47,654 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/691ea56905d2,32909,1731262402364 already deleted, retry=false 2024-11-10T18:16:47,654 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 691ea56905d2,32909,1731262402364 expired; onlineServers=0 2024-11-10T18:16:47,654 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '691ea56905d2,41343,1731262401541' ***** 2024-11-10T18:16:47,654 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T18:16:47,654 INFO [M:0;691ea56905d2:41343 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T18:16:47,655 INFO [M:0;691ea56905d2:41343 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T18:16:47,655 DEBUG [M:0;691ea56905d2:41343 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T18:16:47,655 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T18:16:47,655 DEBUG [M:0;691ea56905d2:41343 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T18:16:47,655 DEBUG [master/691ea56905d2:0:becomeActiveMaster-HFileCleaner.large.0-1731262404742 {}] cleaner.HFileCleaner(306): Exit Thread[master/691ea56905d2:0:becomeActiveMaster-HFileCleaner.large.0-1731262404742,5,FailOnTimeoutGroup] 2024-11-10T18:16:47,655 DEBUG [master/691ea56905d2:0:becomeActiveMaster-HFileCleaner.small.0-1731262404743 {}] cleaner.HFileCleaner(306): Exit Thread[master/691ea56905d2:0:becomeActiveMaster-HFileCleaner.small.0-1731262404743,5,FailOnTimeoutGroup] 2024-11-10T18:16:47,655 INFO [M:0;691ea56905d2:41343 {}] hbase.ChoreService(370): Chore service for: master/691ea56905d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T18:16:47,655 INFO [M:0;691ea56905d2:41343 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T18:16:47,655 DEBUG [M:0;691ea56905d2:41343 {}] master.HMaster(1795): Stopping service threads 2024-11-10T18:16:47,655 INFO [M:0;691ea56905d2:41343 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T18:16:47,655 INFO [M:0;691ea56905d2:41343 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T18:16:47,656 ERROR [M:0;691ea56905d2:41343 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (2068722025) connection to localhost/127.0.0.1:39555 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:39555,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-10T18:16:47,656 INFO [M:0;691ea56905d2:41343 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T18:16:47,656 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T18:16:47,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T18:16:47,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T18:16:47,657 DEBUG [M:0;691ea56905d2:41343 {}] zookeeper.ZKUtil(347): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T18:16:47,657 WARN [M:0;691ea56905d2:41343 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T18:16:47,658 INFO [M:0;691ea56905d2:41343 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/.lastflushedseqids 2024-11-10T18:16:47,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742438_1614 (size=72) 2024-11-10T18:16:47,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T18:16:47,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32909-0x1010315b2cd0001, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T18:16:47,753 INFO [RS:0;691ea56905d2:32909 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T18:16:47,753 INFO [RS:0;691ea56905d2:32909 {}] regionserver.HRegionServer(1031): Exiting; stopping=691ea56905d2,32909,1731262402364; zookeeper connection closed. 2024-11-10T18:16:47,753 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@51a9e53a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@51a9e53a 2024-11-10T18:16:47,754 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-10T18:16:48,067 INFO [M:0;691ea56905d2:41343 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T18:16:48,068 INFO [M:0;691ea56905d2:41343 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T18:16:48,068 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T18:16:48,068 INFO [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:16:48,068 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:16:48,068 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T18:16:48,068 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:16:48,068 INFO [M:0;691ea56905d2:41343 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=756.30 KB heapSize=925.41 KB 2024-11-10T18:16:48,083 DEBUG [M:0;691ea56905d2:41343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/904e107aa4874c75aaabc6fdce43effb is 82, key is hbase:meta,,1/info:regioninfo/1731262406086/Put/seqid=0 2024-11-10T18:16:48,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742439_1615 (size=5672) 2024-11-10T18:16:48,486 INFO [M:0;691ea56905d2:41343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2089 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/904e107aa4874c75aaabc6fdce43effb 2024-11-10T18:16:48,506 DEBUG [M:0;691ea56905d2:41343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/373d24cfab6947ea8d7f2a11ee65664b is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x89/proc:d/1731262573510/Put/seqid=0 2024-11-10T18:16:48,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742440_1616 (size=43610) 2024-11-10T18:16:48,909 INFO [M:0;691ea56905d2:41343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=755.75 KB at sequenceid=2089 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/373d24cfab6947ea8d7f2a11ee65664b 2024-11-10T18:16:48,912 INFO [M:0;691ea56905d2:41343 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 373d24cfab6947ea8d7f2a11ee65664b 2024-11-10T18:16:48,926 DEBUG [M:0;691ea56905d2:41343 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f93a05cc3a841d887208b0115c319d8 is 69, key is 691ea56905d2,32909,1731262402364/rs:state/1731262404804/Put/seqid=0 2024-11-10T18:16:48,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073742441_1617 (size=5156) 2024-11-10T18:16:49,330 INFO [M:0;691ea56905d2:41343 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2089 (bloomFilter=true), to=hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f93a05cc3a841d887208b0115c319d8 2024-11-10T18:16:49,333 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/904e107aa4874c75aaabc6fdce43effb as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/904e107aa4874c75aaabc6fdce43effb 2024-11-10T18:16:49,336 INFO [M:0;691ea56905d2:41343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/904e107aa4874c75aaabc6fdce43effb, entries=8, sequenceid=2089, filesize=5.5 K 2024-11-10T18:16:49,336 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/373d24cfab6947ea8d7f2a11ee65664b as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/373d24cfab6947ea8d7f2a11ee65664b 2024-11-10T18:16:49,339 INFO [M:0;691ea56905d2:41343 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 373d24cfab6947ea8d7f2a11ee65664b 2024-11-10T18:16:49,339 INFO [M:0;691ea56905d2:41343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/373d24cfab6947ea8d7f2a11ee65664b, entries=160, sequenceid=2089, filesize=42.6 K 2024-11-10T18:16:49,339 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8f93a05cc3a841d887208b0115c319d8 as hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8f93a05cc3a841d887208b0115c319d8 2024-11-10T18:16:49,342 INFO [M:0;691ea56905d2:41343 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39555/user/jenkins/test-data/7e9ab316-b7a2-91eb-4fa6-957b3852b193/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8f93a05cc3a841d887208b0115c319d8, entries=1, sequenceid=2089, filesize=5.0 K 2024-11-10T18:16:49,343 INFO [M:0;691ea56905d2:41343 {}] regionserver.HRegion(3140): Finished flush of dataSize ~756.30 KB/774454, heapSize ~925.12 KB/947320, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2089, compaction requested=false 2024-11-10T18:16:49,344 INFO [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T18:16:49,344 DEBUG [M:0;691ea56905d2:41343 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731262608068Disabling compacts and flushes for region at 1731262608068Disabling writes for close at 1731262608068Obtaining lock to block concurrent updates at 1731262608068Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731262608068Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=774454, getHeapSize=947560, getOffHeapSize=0, getCellsCount=2274 at 1731262608068Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731262608069 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731262608069Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731262608083 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731262608083Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731262608490 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731262608505 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731262608505Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731262608912 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731262608926 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731262608926Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a870080: reopening flushed file at 1731262609333 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17b41668: reopening flushed file at 1731262609336 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59f9890f: reopening flushed file at 1731262609339 (+3 ms)Finished flush of dataSize ~756.30 KB/774454, heapSize ~925.12 KB/947320, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2089, compaction requested=false at 1731262609343 (+4 ms)Writing region close event to WAL at 1731262609344 (+1 ms)Closed at 1731262609344 2024-11-10T18:16:49,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39089 is added to blk_1073741830_1006 (size=911398) 2024-11-10T18:16:49,347 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T18:16:49,347 INFO [M:0;691ea56905d2:41343 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T18:16:49,347 INFO [M:0;691ea56905d2:41343 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41343 2024-11-10T18:16:49,347 INFO [M:0;691ea56905d2:41343 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T18:16:49,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T18:16:49,449 INFO [M:0;691ea56905d2:41343 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T18:16:49,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41343-0x1010315b2cd0000, quorum=127.0.0.1:59263, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T18:16:49,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-10T18:16:49,456 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T18:16:49,456 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T18:16:49,456 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T18:16:49,456 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/hadoop.log.dir/,STOPPED} 2024-11-10T18:16:49,459 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-10T18:16:49,459 WARN [BP-1691272945-172.17.0.2-1731262398071 heartbeating to localhost/127.0.0.1:39555 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-10T18:16:49,459 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-10T18:16:49,459 WARN [BP-1691272945-172.17.0.2-1731262398071 heartbeating to localhost/127.0.0.1:39555 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1691272945-172.17.0.2-1731262398071 (Datanode Uuid a583ee22-9a26-4d25-9d41-36495a87b783) service to localhost/127.0.0.1:39555 2024-11-10T18:16:49,461 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/data/data1/current/BP-1691272945-172.17.0.2-1731262398071 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T18:16:49,461 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/cluster_0cee8799-7317-de7b-00a2-4cbc5fb7f4f4/data/data2/current/BP-1691272945-172.17.0.2-1731262398071 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-10T18:16:49,462 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-10T18:16:49,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-10T18:16:49,470 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-10T18:16:49,470 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-10T18:16:49,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-10T18:16:49,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e430f59d-0796-a5d3-ff93-4ed05490be61/hadoop.log.dir/,STOPPED} 2024-11-10T18:16:49,486 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T18:16:49,611 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down